From 3fef393c3c4acbd15ed421246e142c2eec044d7e Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Thu, 10 Mar 2022 21:49:13 -0800 Subject: [PATCH 01/14] SQLite support Signed-off-by: Ketan Umare --- pkg/repositories/database.go | 14 +++++++---- pkg/repositories/database_test.go | 23 +++++++++++++++++++ .../interfaces/application_configuration.go | 10 ++++++-- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/pkg/repositories/database.go b/pkg/repositories/database.go index e4dd13cc3..92bf7e06d 100644 --- a/pkg/repositories/database.go +++ b/pkg/repositories/database.go @@ -3,6 +3,7 @@ package repositories import ( "context" "fmt" + "gorm.io/driver/sqlite" "io/ioutil" "os" "strings" @@ -79,10 +80,13 @@ func GetDB(ctx context.Context, dbConfig *runtimeInterfaces.DbConfig, logConfig logLevel := getGormLogLevel(ctx, logConfig) switch { - // TODO: Figure out a better proxy for a non-empty postgres config - case len(dbConfig.PostgresConfig.Host) > 0 || len(dbConfig.PostgresConfig.User) > 0 || len(dbConfig.PostgresConfig.DbName) > 0: - dialector = postgres.Open(getPostgresDsn(ctx, dbConfig.PostgresConfig)) - // TODO: add other gorm-supported db type handling in further case blocks. + case dbConfig.SQLiteConfig != nil: + if dbConfig.SQLiteConfig.File == "" { + return nil, fmt.Errorf("illegal sqlite database configuration. `file` is a required parameter and should be a path") + } + dialector = sqlite.Open(dbConfig.SQLiteConfig.File) + case dbConfig.PostgresConfig != nil && (len(dbConfig.PostgresConfig.Host) > 0 || len(dbConfig.PostgresConfig.User) > 0 || len(dbConfig.PostgresConfig.DbName) > 0): + dialector = postgres.Open(getPostgresDsn(ctx, *dbConfig.PostgresConfig)) case len(dbConfig.DeprecatedHost) > 0 || len(dbConfig.DeprecatedUser) > 0 || len(dbConfig.DeprecatedDbName) > 0: pgConfig := runtimeInterfaces.PostgresConfig{ Host: dbConfig.DeprecatedHost, @@ -96,7 +100,7 @@ func GetDB(ctx context.Context, dbConfig *runtimeInterfaces.DbConfig, logConfig } dialector = postgres.Open(getPostgresDsn(ctx, pgConfig)) default: - panic(fmt.Sprintf("Unrecognized database config %v", dbConfig)) + return nil, fmt.Errorf("unrecognized database config, %v. Supported only postgres and sqlite", dbConfig) } gormDb, err := gorm.Open(dialector, &gorm.Config{ Logger: gormLogger.Default.LogMode(logLevel), diff --git a/pkg/repositories/database_test.go b/pkg/repositories/database_test.go index 5b1de474e..40e564a71 100644 --- a/pkg/repositories/database_test.go +++ b/pkg/repositories/database_test.go @@ -4,6 +4,7 @@ import ( "context" "io/ioutil" "os" + "path" "path/filepath" "testing" "time" @@ -143,3 +144,25 @@ func TestSetupDbConnectionPool(t *testing.T) { assert.NotNil(t, err) }) } + +func TestGetDB(t *testing.T) { + ctx := context.TODO() + + t.Run("missing DB Config", func(t *testing.T) { + _, err := GetDB(ctx, &runtimeInterfaces.DbConfig{}, &logger.Config{}) + assert.Error(t, err) + }) + + t.Run("sqlite config", func(t *testing.T) { + dbFile := path.Join(t.TempDir(), "admin.db") + db, err := GetDB(ctx, &runtimeInterfaces.DbConfig{ + SQLiteConfig: &runtimeInterfaces.SQLiteConfig{ + File: dbFile, + }, + }, &logger.Config{}) + assert.NoError(t, err) + assert.NotNil(t, db) + assert.FileExists(t, dbFile) + assert.Equal(t, "sqlite", db.Name()) + }) +} diff --git a/pkg/runtime/interfaces/application_configuration.go b/pkg/runtime/interfaces/application_configuration.go index 2eca76c81..544545ada 100644 --- a/pkg/runtime/interfaces/application_configuration.go +++ b/pkg/runtime/interfaces/application_configuration.go @@ -22,7 +22,13 @@ type DbConfig struct { MaxIdleConnections int `json:"maxIdleConnections" pflag:",maxIdleConnections sets the maximum number of connections in the idle connection pool."` MaxOpenConnections int `json:"maxOpenConnections" pflag:",maxOpenConnections sets the maximum number of open connections to the database."` ConnMaxLifeTime config.Duration `json:"connMaxLifeTime" pflag:",sets the maximum amount of time a connection may be reused"` - PostgresConfig PostgresConfig `json:"postgres"` + PostgresConfig *PostgresConfig `json:"postgres,omitempty"` + SQLiteConfig *SQLiteConfig `json:"sqlite,omitempty"` +} + +// SQLiteConfig can be used to configure +type SQLiteConfig struct { + File string `json:"file" pflag:",The path to the file (existing or new) where the DB should be created / stored. If existing, then this will be re-used, else a new will be created"` } // PostgresConfig includes specific config options for opening a connection to a postgres database. @@ -38,7 +44,7 @@ type PostgresConfig struct { Debug bool `json:"debug" pflag:" Whether or not to start the database connection with debug mode enabled."` } -// This configuration is the base configuration to start admin +// ApplicationConfig is the base configuration to start admin type ApplicationConfig struct { // The RoleName key inserted as an annotation (https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) // in Flyte Workflow CRDs created in the CreateExecution flow. The corresponding role value is defined in the From e1e99fec36ab299c6346fee8bcba9e6d93c7468c Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Thu, 10 Mar 2022 22:54:43 -0800 Subject: [PATCH 02/14] Move migrate commands Signed-off-by: Ketan Umare --- .gitignore | 1 + cmd/entrypoints/clusterresource.go | 5 +- cmd/entrypoints/migrate.go | 100 +---- cmd/entrypoints/serve.go | 376 +----------------- pkg/rpc/adminservice/base.go | 10 - .../interfaces/application_configuration.go | 1 + pkg/server/initialize.go | 76 ++++ pkg/server/service.go | 365 +++++++++++++++++ 8 files changed, 467 insertions(+), 467 deletions(-) create mode 100644 pkg/server/initialize.go create mode 100644 pkg/server/service.go diff --git a/.gitignore b/.gitignore index f24947dc5..0178cf5fe 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ node_modules/ .virtualgo boilerplate/lyft/end2end/tmp +dist diff --git a/cmd/entrypoints/clusterresource.go b/cmd/entrypoints/clusterresource.go index ab63b3377..81653d7b4 100644 --- a/cmd/entrypoints/clusterresource.go +++ b/cmd/entrypoints/clusterresource.go @@ -2,6 +2,7 @@ package entrypoints import ( "context" + errors2 "github.com/pkg/errors" "github.com/flyteorg/flyteadmin/pkg/repositories/errors" @@ -88,14 +89,14 @@ var controllerRunCmd = &cobra.Command{ var controllerSyncCmd = &cobra.Command{ Use: "sync", Short: "This command will sync cluster resources", - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() configuration := runtime.NewConfigurationProvider() scope := promutils.NewScope(configuration.ApplicationConfiguration().GetTopLevelConfig().MetricsScope).NewSubScope("clusterresource") clusterResourceController := getClusterResourceController(ctx, scope, configuration) err := clusterResourceController.Sync(ctx) if err != nil { - logger.Fatalf(ctx, "Failed to sync cluster resources [%+v]", err) + return errors2.Wrap(err, "Failed to sync cluster resources ") } logger.Infof(ctx, "ClusterResourceController synced successfully") }, diff --git a/cmd/entrypoints/migrate.go b/cmd/entrypoints/migrate.go index fdad8c7fc..313e1c52b 100644 --- a/cmd/entrypoints/migrate.go +++ b/cmd/entrypoints/migrate.go @@ -2,13 +2,8 @@ package entrypoints import ( "context" + "github.com/flyteorg/flyteadmin/pkg/server" - "github.com/flyteorg/flyteadmin/pkg/repositories" - - "github.com/flyteorg/flyteadmin/pkg/repositories/config" - "github.com/flyteorg/flyteadmin/pkg/runtime" - "github.com/flyteorg/flytestdlib/logger" - "github.com/go-gormigrate/gormigrate/v2" "github.com/spf13/cobra" _ "gorm.io/driver/postgres" // Required to import database driver. ) @@ -22,36 +17,9 @@ var parentMigrateCmd = &cobra.Command{ var migrateCmd = &cobra.Command{ Use: "run", Short: "This command will run all the migrations for the database", - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() - configuration := runtime.NewConfigurationProvider() - databaseConfig := configuration.ApplicationConfiguration().GetDbConfig() - logConfig := logger.GetConfig() - - db, err := repositories.GetDB(ctx, databaseConfig, logConfig) - if err != nil { - logger.Fatal(ctx, err) - } - - sqlDB, err := db.DB() - if err != nil { - logger.Fatal(ctx, err) - } - - defer func(deferCtx context.Context) { - if err = sqlDB.Close(); err != nil { - logger.Fatal(deferCtx, err) - } - }(ctx) - - if err = sqlDB.Ping(); err != nil { - logger.Fatal(ctx, err) - } - m := gormigrate.New(db, gormigrate.DefaultOptions, config.Migrations) - if err = m.Migrate(); err != nil { - logger.Fatalf(ctx, "Could not migrate: %v", err) - } - logger.Infof(ctx, "Migration ran successfully") + return server.Migrate(ctx) }, } @@ -59,36 +27,9 @@ var migrateCmd = &cobra.Command{ var rollbackCmd = &cobra.Command{ Use: "rollback", Short: "This command will rollback one migration", - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() - configuration := runtime.NewConfigurationProvider() - databaseConfig := configuration.ApplicationConfiguration().GetDbConfig() - logConfig := logger.GetConfig() - - db, err := repositories.GetDB(ctx, databaseConfig, logConfig) - if err != nil { - logger.Fatal(ctx, err) - } - sqlDB, err := db.DB() - if err != nil { - logger.Fatal(ctx, err) - } - defer func(deferCtx context.Context) { - if err = sqlDB.Close(); err != nil { - logger.Fatal(deferCtx, err) - } - }(ctx) - - if err = sqlDB.Ping(); err != nil { - logger.Fatal(ctx, err) - } - - m := gormigrate.New(db, gormigrate.DefaultOptions, config.Migrations) - err = m.RollbackLast() - if err != nil { - logger.Fatalf(ctx, "Could not rollback latest migration: %v", err) - } - logger.Infof(ctx, "Rolled back one migration successfully") + return server.Rollback(ctx) }, } @@ -96,36 +37,9 @@ var rollbackCmd = &cobra.Command{ var seedProjectsCmd = &cobra.Command{ Use: "seed-projects", Short: "Seed projects in the database.", - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() - configuration := runtime.NewConfigurationProvider() - databaseConfig := configuration.ApplicationConfiguration().GetDbConfig() - logConfig := logger.GetConfig() - - db, err := repositories.GetDB(ctx, databaseConfig, logConfig) - if err != nil { - logger.Fatal(ctx, err) - } - - sqlDB, err := db.DB() - if err != nil { - logger.Fatal(ctx, err) - } - - defer func(deferCtx context.Context) { - if err = sqlDB.Close(); err != nil { - logger.Fatal(deferCtx, err) - } - }(ctx) - - if err = sqlDB.Ping(); err != nil { - logger.Fatal(ctx, err) - } - - if err = config.SeedProjects(db, args); err != nil { - logger.Fatalf(ctx, "Could not add projects to database with err: %v", err) - } - logger.Infof(ctx, "Successfully added projects to database") + return server.SeedProjects(ctx, args) }, } diff --git a/cmd/entrypoints/serve.go b/cmd/entrypoints/serve.go index 2028dfeb9..cad0b70b1 100644 --- a/cmd/entrypoints/serve.go +++ b/cmd/entrypoints/serve.go @@ -2,66 +2,36 @@ package entrypoints import ( "context" - "crypto/tls" - - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/secretmanager" - - authConfig "github.com/flyteorg/flyteadmin/auth/config" - - "github.com/flyteorg/flyteadmin/auth/authzserver" - - "github.com/gorilla/handlers" - - "github.com/flyteorg/flyteadmin/auth" - "github.com/flyteorg/flyteadmin/auth/interfaces" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - grpcauth "github.com/grpc-ecosystem/go-grpc-middleware/auth" - - "net" - "net/http" - _ "net/http/pprof" // Required to serve application. - "strings" - - "github.com/flyteorg/flyteadmin/pkg/server" - "github.com/pkg/errors" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/health" - "google.golang.org/grpc/health/grpc_health_v1" + "github.com/flyteorg/flytestdlib/profutils" "github.com/flyteorg/flyteadmin/pkg/common" - flyteService "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service" + "github.com/flyteorg/flyteadmin/pkg/server" "github.com/flyteorg/flytestdlib/logger" - "github.com/grpc-ecosystem/grpc-gateway/runtime" - - "github.com/flyteorg/flyteadmin/pkg/config" - "github.com/flyteorg/flyteadmin/pkg/rpc/adminservice" "github.com/spf13/cobra" + _ "net/http/pprof" // Required to serve application. + runtimeConfig "github.com/flyteorg/flyteadmin/pkg/runtime" "github.com/flyteorg/flytestdlib/contextutils" "github.com/flyteorg/flytestdlib/promutils/labeled" - grpcPrometheus "github.com/grpc-ecosystem/go-grpc-prometheus" - "google.golang.org/grpc" - "google.golang.org/grpc/reflection" ) -var defaultCorsHeaders = []string{"Content-Type"} - // serveCmd represents the serve command var serveCmd = &cobra.Command{ Use: "serve", Short: "Launches the Flyte admin server", RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() - serverConfig := config.GetConfig() - - if serverConfig.Security.Secure { - return serveGatewaySecure(ctx, serverConfig, authConfig.GetConfig()) - } + // Serve profiling endpoints. + cfg := runtimeConfig.NewConfigurationProvider() + go func() { + err := profutils.StartProfilingServerWithDefaultHandlers( + ctx, cfg.ApplicationConfiguration().GetTopLevelConfig().GetProfilerPort(), nil) + if err != nil { + logger.Panicf(ctx, "Failed to Start profiling and Metrics server. Error, %v", err) + } + }() - return serveGatewayInsecure(ctx, serverConfig, authConfig.GetConfig()) + return server.Serve(ctx) }, } @@ -76,321 +46,3 @@ func init() { contextutils.TaskTypeKey, common.RuntimeTypeKey, common.RuntimeVersionKey) } -func blanketAuthorization(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) ( - resp interface{}, err error) { - - identityContext := auth.IdentityContextFromContext(ctx) - if identityContext.IsEmpty() { - return handler(ctx, req) - } - - if !identityContext.Scopes().Has(auth.ScopeAll) { - return nil, status.Errorf(codes.Unauthenticated, "authenticated user doesn't have required scope") - } - - return handler(ctx, req) -} - -// Creates a new gRPC Server with all the configuration -func newGRPCServer(ctx context.Context, cfg *config.ServerConfig, authCtx interfaces.AuthenticationContext, - opts ...grpc.ServerOption) (*grpc.Server, error) { - // Not yet implemented for streaming - var chainedUnaryInterceptors grpc.UnaryServerInterceptor - if cfg.Security.UseAuth { - logger.Infof(ctx, "Creating gRPC server with authentication") - chainedUnaryInterceptors = grpc_middleware.ChainUnaryServer(grpcPrometheus.UnaryServerInterceptor, - auth.GetAuthenticationCustomMetadataInterceptor(authCtx), - grpcauth.UnaryServerInterceptor(auth.GetAuthenticationInterceptor(authCtx)), - auth.AuthenticationLoggingInterceptor, - blanketAuthorization, - ) - } else { - logger.Infof(ctx, "Creating gRPC server without authentication") - chainedUnaryInterceptors = grpc_middleware.ChainUnaryServer(grpcPrometheus.UnaryServerInterceptor) - } - - serverOpts := []grpc.ServerOption{ - grpc.StreamInterceptor(grpcPrometheus.StreamServerInterceptor), - grpc.UnaryInterceptor(chainedUnaryInterceptors), - } - if cfg.GrpcConfig.MaxMessageSizeBytes > 0 { - serverOpts = append(serverOpts, grpc.MaxRecvMsgSize(cfg.GrpcConfig.MaxMessageSizeBytes)) - } - serverOpts = append(serverOpts, opts...) - grpcServer := grpc.NewServer(serverOpts...) - grpcPrometheus.Register(grpcServer) - flyteService.RegisterAdminServiceServer(grpcServer, adminservice.NewAdminServer(ctx, cfg.KubeConfig, cfg.Master)) - if cfg.Security.UseAuth { - flyteService.RegisterAuthMetadataServiceServer(grpcServer, authCtx.AuthMetadataService()) - flyteService.RegisterIdentityServiceServer(grpcServer, authCtx.IdentityService()) - } - - healthServer := health.NewServer() - healthServer.SetServingStatus("flyteadmin", grpc_health_v1.HealthCheckResponse_SERVING) - grpc_health_v1.RegisterHealthServer(grpcServer, healthServer) - if cfg.GrpcConfig.ServerReflection || cfg.GrpcServerReflection { - reflection.Register(grpcServer) - } - return grpcServer, nil -} - -func GetHandleOpenapiSpec(ctx context.Context) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - swaggerBytes, err := flyteService.Asset("admin.swagger.json") - if err != nil { - logger.Warningf(ctx, "Err %v", err) - w.WriteHeader(http.StatusFailedDependency) - } else { - w.WriteHeader(http.StatusOK) - _, err := w.Write(swaggerBytes) - if err != nil { - logger.Errorf(ctx, "failed to write openAPI information, error: %s", err.Error()) - } - } - } -} - -func healthCheckFunc(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) -} - -func newHTTPServer(ctx context.Context, cfg *config.ServerConfig, authCfg *authConfig.Config, authCtx interfaces.AuthenticationContext, - grpcAddress string, grpcConnectionOpts ...grpc.DialOption) (*http.ServeMux, error) { - - // Register the server that will serve HTTP/REST Traffic - mux := http.NewServeMux() - - // Register healthcheck - mux.HandleFunc("/healthcheck", healthCheckFunc) - - // Register OpenAPI endpoint - // This endpoint will serve the OpenAPI2 spec generated by the swagger protoc plugin, and bundled by go-bindata - mux.HandleFunc("/api/v1/openapi", GetHandleOpenapiSpec(ctx)) - - var gwmuxOptions = make([]runtime.ServeMuxOption, 0) - // This option means that http requests are served with protobufs, instead of json. We always want this. - gwmuxOptions = append(gwmuxOptions, runtime.WithMarshalerOption("application/octet-stream", &runtime.ProtoMarshaller{})) - - if cfg.Security.UseAuth { - // Add HTTP handlers for OIDC endpoints - auth.RegisterHandlers(ctx, mux, authCtx) - - // Add HTTP handlers for OAuth2 endpoints - authzserver.RegisterHandlers(mux, authCtx) - - // This option translates HTTP authorization data (cookies) into a gRPC metadata field - gwmuxOptions = append(gwmuxOptions, runtime.WithMetadata(auth.GetHTTPRequestCookieToMetadataHandler(authCtx))) - - // In an attempt to be able to selectively enforce whether or not authentication is required, we're going to tag - // the requests that come from the HTTP gateway. See the enforceHttp/Grpc options for more information. - gwmuxOptions = append(gwmuxOptions, runtime.WithMetadata(auth.GetHTTPMetadataTaggingHandler())) - } - - // Create the grpc-gateway server with the options specified - gwmux := runtime.NewServeMux(gwmuxOptions...) - - err := flyteService.RegisterAdminServiceHandlerFromEndpoint(ctx, gwmux, grpcAddress, grpcConnectionOpts) - if err != nil { - return nil, errors.Wrap(err, "error registering admin service") - } - - err = flyteService.RegisterAuthMetadataServiceHandlerFromEndpoint(ctx, gwmux, grpcAddress, grpcConnectionOpts) - if err != nil { - return nil, errors.Wrap(err, "error registering auth service") - } - - err = flyteService.RegisterIdentityServiceHandlerFromEndpoint(ctx, gwmux, grpcAddress, grpcConnectionOpts) - if err != nil { - return nil, errors.Wrap(err, "error registering identity service") - } - - mux.Handle("/", gwmux) - - return mux, nil -} - -func serveGatewayInsecure(ctx context.Context, cfg *config.ServerConfig, authCfg *authConfig.Config) error { - logger.Infof(ctx, "Serving Flyte Admin Insecure") - - // This will parse configuration and create the necessary objects for dealing with auth - var authCtx interfaces.AuthenticationContext - var err error - // This code is here to support authentication without SSL. This setup supports a network topology where - // Envoy does the SSL termination. The final hop is made over localhost only on a trusted machine. - // Warning: Running authentication without SSL in any other topology is a severe security flaw. - // See the auth.Config object for additional settings as well. - if cfg.Security.UseAuth { - sm := secretmanager.NewFileEnvSecretManager(secretmanager.GetConfig()) - var oauth2Provider interfaces.OAuth2Provider - var oauth2ResourceServer interfaces.OAuth2ResourceServer - if authCfg.AppAuth.AuthServerType == authConfig.AuthorizationServerTypeSelf { - oauth2Provider, err = authzserver.NewProvider(ctx, authCfg.AppAuth.SelfAuthServer, sm) - if err != nil { - logger.Errorf(ctx, "Error creating authorization server %s", err) - return err - } - - oauth2ResourceServer = oauth2Provider - } else { - oauth2ResourceServer, err = authzserver.NewOAuth2ResourceServer(ctx, authCfg.AppAuth.ExternalAuthServer, authCfg.UserAuth.OpenID.BaseURL) - if err != nil { - logger.Errorf(ctx, "Error creating resource server %s", err) - return err - } - } - - oauth2MetadataProvider := authzserver.NewService(authCfg) - oidcUserInfoProvider := auth.NewUserInfoProvider() - - authCtx, err = auth.NewAuthenticationContext(ctx, sm, oauth2Provider, oauth2ResourceServer, oauth2MetadataProvider, oidcUserInfoProvider, authCfg) - if err != nil { - logger.Errorf(ctx, "Error creating auth context %s", err) - return err - } - } - - grpcServer, err := newGRPCServer(ctx, cfg, authCtx) - if err != nil { - return errors.Wrap(err, "failed to create GRPC server") - } - - logger.Infof(ctx, "Serving GRPC Traffic on: %s", cfg.GetGrpcHostAddress()) - lis, err := net.Listen("tcp", cfg.GetGrpcHostAddress()) - if err != nil { - return errors.Wrapf(err, "failed to listen on GRPC port: %s", cfg.GetGrpcHostAddress()) - } - - go func() { - err := grpcServer.Serve(lis) - logger.Fatalf(ctx, "Failed to create GRPC Server, Err: ", err) - }() - - logger.Infof(ctx, "Starting HTTP/1 Gateway server on %s", cfg.GetHostAddress()) - grpcOptions := []grpc.DialOption{ - grpc.WithInsecure(), - grpc.WithMaxHeaderListSize(common.MaxResponseStatusBytes), - } - if cfg.GrpcConfig.MaxMessageSizeBytes > 0 { - grpcOptions = append(grpcOptions, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(cfg.GrpcConfig.MaxMessageSizeBytes))) - } - httpServer, err := newHTTPServer(ctx, cfg, authCfg, authCtx, cfg.GetGrpcHostAddress(), grpcOptions...) - if err != nil { - return err - } - - var handler http.Handler - if cfg.Security.AllowCors { - handler = handlers.CORS( - handlers.AllowCredentials(), - handlers.AllowedOrigins(cfg.Security.AllowedOrigins), - handlers.AllowedHeaders(append(defaultCorsHeaders, cfg.Security.AllowedHeaders...)), - handlers.AllowedMethods([]string{"GET", "POST", "DELETE", "HEAD", "PUT", "PATCH"}), - )(httpServer) - } else { - handler = httpServer - } - - err = http.ListenAndServe(cfg.GetHostAddress(), handler) - if err != nil { - return errors.Wrapf(err, "failed to Start HTTP Server") - } - - return nil -} - -// grpcHandlerFunc returns an http.Handler that delegates to grpcServer on incoming gRPC -// connections or otherHandler otherwise. -// See https://github.com/philips/grpc-gateway-example/blob/master/cmd/serve.go for reference -func grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // This is a partial recreation of gRPC's internal checks - if r.ProtoMajor == 2 && strings.Contains(r.Header.Get("Content-Type"), "application/grpc") { - grpcServer.ServeHTTP(w, r) - } else { - otherHandler.ServeHTTP(w, r) - } - }) -} - -func serveGatewaySecure(ctx context.Context, cfg *config.ServerConfig, authCfg *authConfig.Config) error { - certPool, cert, err := server.GetSslCredentials(ctx, cfg.Security.Ssl.CertificateFile, cfg.Security.Ssl.KeyFile) - if err != nil { - return err - } - // This will parse configuration and create the necessary objects for dealing with auth - var authCtx interfaces.AuthenticationContext - if cfg.Security.UseAuth { - sm := secretmanager.NewFileEnvSecretManager(secretmanager.GetConfig()) - var oauth2Provider interfaces.OAuth2Provider - var oauth2ResourceServer interfaces.OAuth2ResourceServer - if authCfg.AppAuth.AuthServerType == authConfig.AuthorizationServerTypeSelf { - oauth2Provider, err = authzserver.NewProvider(ctx, authCfg.AppAuth.SelfAuthServer, sm) - if err != nil { - logger.Errorf(ctx, "Error creating authorization server %s", err) - return err - } - - oauth2ResourceServer = oauth2Provider - } else { - oauth2ResourceServer, err = authzserver.NewOAuth2ResourceServer(ctx, authCfg.AppAuth.ExternalAuthServer, authCfg.UserAuth.OpenID.BaseURL) - if err != nil { - logger.Errorf(ctx, "Error creating resource server %s", err) - return err - } - } - - oauth2MetadataProvider := authzserver.NewService(authCfg) - oidcUserInfoProvider := auth.NewUserInfoProvider() - - authCtx, err = auth.NewAuthenticationContext(ctx, sm, oauth2Provider, oauth2ResourceServer, oauth2MetadataProvider, oidcUserInfoProvider, authCfg) - if err != nil { - logger.Errorf(ctx, "Error creating auth context %s", err) - return err - } - } - - grpcServer, err := newGRPCServer(ctx, cfg, authCtx, - grpc.Creds(credentials.NewServerTLSFromCert(cert))) - if err != nil { - return errors.Wrap(err, "failed to create GRPC server") - } - - // Whatever certificate is used, pass it along for easier development - dialCreds := credentials.NewTLS(&tls.Config{ - ServerName: cfg.GetHostAddress(), - RootCAs: certPool, - }) - serverOpts := []grpc.DialOption{ - grpc.WithTransportCredentials(dialCreds), - } - if cfg.GrpcConfig.MaxMessageSizeBytes > 0 { - serverOpts = append(serverOpts, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(cfg.GrpcConfig.MaxMessageSizeBytes))) - } - httpServer, err := newHTTPServer(ctx, cfg, authCfg, authCtx, cfg.GetHostAddress(), serverOpts...) - if err != nil { - return err - } - - conn, err := net.Listen("tcp", cfg.GetHostAddress()) - if err != nil { - panic(err) - } - - srv := &http.Server{ - Addr: cfg.GetHostAddress(), - Handler: grpcHandlerFunc(grpcServer, httpServer), - TLSConfig: &tls.Config{ - Certificates: []tls.Certificate{*cert}, - NextProtos: []string{"h2"}, - }, - } - - err = srv.Serve(tls.NewListener(conn, srv.TLSConfig)) - - if err != nil { - return errors.Wrapf(err, "failed to Start HTTP/2 Server") - } - return nil -} diff --git a/pkg/rpc/adminservice/base.go b/pkg/rpc/adminservice/base.go index 891f8f9a5..f3f740060 100644 --- a/pkg/rpc/adminservice/base.go +++ b/pkg/rpc/adminservice/base.go @@ -24,7 +24,6 @@ import ( "github.com/flyteorg/flyteadmin/pkg/workflowengine" workflowengineImpl "github.com/flyteorg/flyteadmin/pkg/workflowengine/impl" "github.com/flyteorg/flytestdlib/logger" - "github.com/flyteorg/flytestdlib/profutils" "github.com/flyteorg/flytestdlib/promutils" "github.com/flyteorg/flytestdlib/storage" "github.com/golang/protobuf/proto" @@ -155,15 +154,6 @@ func NewAdminServer(ctx context.Context, kubeConfig, master string) *AdminServic scheduledWorkflowExecutor.Run() }() - // Serve profiling endpoints. - go func() { - err := profutils.StartProfilingServerWithDefaultHandlers( - ctx, applicationConfiguration.GetProfilerPort(), nil) - if err != nil { - logger.Panicf(ctx, "Failed to Start profiling and Metrics server. Error, %v", err) - } - }() - nodeExecutionEventWriter := eventWriter.NewNodeExecutionEventWriter(repo, applicationConfiguration.GetAsyncEventsBufferSize()) go func() { nodeExecutionEventWriter.Run() diff --git a/pkg/runtime/interfaces/application_configuration.go b/pkg/runtime/interfaces/application_configuration.go index 544545ada..67731aa83 100644 --- a/pkg/runtime/interfaces/application_configuration.go +++ b/pkg/runtime/interfaces/application_configuration.go @@ -6,6 +6,7 @@ import ( "golang.org/x/time/rate" ) + // DbConfig is used to for initiating the database connection with the store that holds registered // entities (e.g. workflows, tasks, launch plans...) type DbConfig struct { diff --git a/pkg/server/initialize.go b/pkg/server/initialize.go new file mode 100644 index 000000000..2352daa37 --- /dev/null +++ b/pkg/server/initialize.go @@ -0,0 +1,76 @@ +package server + +import ( + "context" + "fmt" + "github.com/flyteorg/flyteadmin/pkg/repositories" + "github.com/flyteorg/flyteadmin/pkg/repositories/config" + "github.com/flyteorg/flyteadmin/pkg/runtime" + "github.com/flyteorg/flytestdlib/logger" + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +func withDB(ctx context.Context, do func(db *gorm.DB) error) error { + configuration := runtime.NewConfigurationProvider() + databaseConfig := configuration.ApplicationConfiguration().GetDbConfig() + logConfig := logger.GetConfig() + + db, err := repositories.GetDB(ctx, databaseConfig, logConfig) + if err != nil { + logger.Fatal(ctx, err) + } + + sqlDB, err := db.DB() + if err != nil { + logger.Fatal(ctx, err) + } + + defer func(deferCtx context.Context) { + if err = sqlDB.Close(); err != nil { + logger.Fatal(deferCtx, err) + } + }(ctx) + + if err = sqlDB.Ping(); err != nil { + return err + } + + return do(db) +} + +// Migrate runs all configured migrations +func Migrate(ctx context.Context) error { + return withDB(ctx, func(db *gorm.DB) error { + m := gormigrate.New(db, gormigrate.DefaultOptions, config.Migrations) + if err := m.Migrate(); err != nil { + return fmt.Errorf("database migration failed: %v", err) + } + logger.Infof(ctx, "Migration ran successfully") + return nil + }) +} + +// Rollback rolls back the last migration +func Rollback(ctx context.Context) error { + return withDB(ctx, func(db *gorm.DB) error { + m := gormigrate.New(db, gormigrate.DefaultOptions, config.Migrations) + err := m.RollbackLast() + if err != nil { + return fmt.Errorf("could not rollback latest migration: %v", err) + } + logger.Infof(ctx, "Rolled back one migration successfully") + return nil + }) +} + +// SeedProjects creates a set of given projects in the DB +func SeedProjects(ctx context.Context, projects []string) error { + return withDB(ctx, func(db *gorm.DB) error { + if err := config.SeedProjects(db, projects); err != nil { + return fmt.Errorf("could not add projects to database with err: %v", err) + } + logger.Infof(ctx, "Successfully added projects to database") + return nil + }) +} diff --git a/pkg/server/service.go b/pkg/server/service.go new file mode 100644 index 000000000..5c683a67c --- /dev/null +++ b/pkg/server/service.go @@ -0,0 +1,365 @@ +package server + +import ( + "context" + "crypto/tls" + "github.com/flyteorg/flyteadmin/auth" + "github.com/flyteorg/flyteadmin/auth/authzserver" + authConfig "github.com/flyteorg/flyteadmin/auth/config" + "github.com/flyteorg/flyteadmin/auth/interfaces" + "github.com/flyteorg/flyteadmin/pkg/common" + "github.com/flyteorg/flyteadmin/pkg/config" + "github.com/flyteorg/flyteadmin/pkg/rpc/adminservice" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service" + "github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/secretmanager" + "github.com/flyteorg/flytestdlib/logger" + "github.com/gorilla/handlers" + "github.com/grpc-ecosystem/go-grpc-middleware" + "github.com/grpc-ecosystem/go-grpc-middleware/auth" + "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/grpc-ecosystem/grpc-gateway/runtime" + "github.com/pkg/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/health" + "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/reflection" + "google.golang.org/grpc/status" + "net" + "net/http" + "strings" +) + +var defaultCorsHeaders = []string{"Content-Type"} + +// Serve starts a server and blocks the calling goroutine +func Serve(ctx context.Context) error { + serverConfig := config.GetConfig() + authCfg := authConfig.GetConfig() + + if serverConfig.Security.Secure { + return serveGatewaySecure(ctx, serverConfig, authCfg) + } + + return serveGatewayInsecure(ctx, serverConfig, authCfg) +} + +func blanketAuthorization(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) ( + resp interface{}, err error) { + + identityContext := auth.IdentityContextFromContext(ctx) + if identityContext.IsEmpty() { + return handler(ctx, req) + } + + if !identityContext.Scopes().Has(auth.ScopeAll) { + return nil, status.Errorf(codes.Unauthenticated, "authenticated user doesn't have required scope") + } + + return handler(ctx, req) +} + +// Creates a new gRPC Server with all the configuration +func newGRPCServer(ctx context.Context, cfg *config.ServerConfig, authCtx interfaces.AuthenticationContext, + opts ...grpc.ServerOption) (*grpc.Server, error) { + // Not yet implemented for streaming + var chainedUnaryInterceptors grpc.UnaryServerInterceptor + if cfg.Security.UseAuth { + logger.Infof(ctx, "Creating gRPC server with authentication") + chainedUnaryInterceptors = grpc_middleware.ChainUnaryServer(grpc_prometheus.UnaryServerInterceptor, + auth.GetAuthenticationCustomMetadataInterceptor(authCtx), + grpc_auth.UnaryServerInterceptor(auth.GetAuthenticationInterceptor(authCtx)), + auth.AuthenticationLoggingInterceptor, + blanketAuthorization, + ) + } else { + logger.Infof(ctx, "Creating gRPC server without authentication") + chainedUnaryInterceptors = grpc_middleware.ChainUnaryServer(grpc_prometheus.UnaryServerInterceptor) + } + + serverOpts := []grpc.ServerOption{ + grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor), + grpc.UnaryInterceptor(chainedUnaryInterceptors), + } + if cfg.GrpcConfig.MaxMessageSizeBytes > 0 { + serverOpts = append(serverOpts, grpc.MaxRecvMsgSize(cfg.GrpcConfig.MaxMessageSizeBytes)) + } + serverOpts = append(serverOpts, opts...) + grpcServer := grpc.NewServer(serverOpts...) + grpc_prometheus.Register(grpcServer) + service.RegisterAdminServiceServer(grpcServer, adminservice.NewAdminServer(ctx, cfg.KubeConfig, cfg.Master)) + if cfg.Security.UseAuth { + service.RegisterAuthMetadataServiceServer(grpcServer, authCtx.AuthMetadataService()) + service.RegisterIdentityServiceServer(grpcServer, authCtx.IdentityService()) + } + + healthServer := health.NewServer() + healthServer.SetServingStatus("flyteadmin", grpc_health_v1.HealthCheckResponse_SERVING) + grpc_health_v1.RegisterHealthServer(grpcServer, healthServer) + if cfg.GrpcConfig.ServerReflection || cfg.GrpcServerReflection { + reflection.Register(grpcServer) + } + return grpcServer, nil +} + +func GetHandleOpenapiSpec(ctx context.Context) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + swaggerBytes, err := service.Asset("admin.swagger.json") + if err != nil { + logger.Warningf(ctx, "Err %v", err) + w.WriteHeader(http.StatusFailedDependency) + } else { + w.WriteHeader(http.StatusOK) + _, err := w.Write(swaggerBytes) + if err != nil { + logger.Errorf(ctx, "failed to write openAPI information, error: %s", err.Error()) + } + } + } +} + +func healthCheckFunc(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) +} + +func newHTTPServer(ctx context.Context, cfg *config.ServerConfig, _ *authConfig.Config, authCtx interfaces.AuthenticationContext, + grpcAddress string, grpcConnectionOpts ...grpc.DialOption) (*http.ServeMux, error) { + + // Register the server that will serve HTTP/REST Traffic + mux := http.NewServeMux() + + // Register healthcheck + mux.HandleFunc("/healthcheck", healthCheckFunc) + + // Register OpenAPI endpoint + // This endpoint will serve the OpenAPI2 spec generated by the swagger protoc plugin, and bundled by go-bindata + mux.HandleFunc("/api/v1/openapi", GetHandleOpenapiSpec(ctx)) + + var gwmuxOptions = make([]runtime.ServeMuxOption, 0) + // This option means that http requests are served with protobufs, instead of json. We always want this. + gwmuxOptions = append(gwmuxOptions, runtime.WithMarshalerOption("application/octet-stream", &runtime.ProtoMarshaller{})) + + if cfg.Security.UseAuth { + // Add HTTP handlers for OIDC endpoints + auth.RegisterHandlers(ctx, mux, authCtx) + + // Add HTTP handlers for OAuth2 endpoints + authzserver.RegisterHandlers(mux, authCtx) + + // This option translates HTTP authorization data (cookies) into a gRPC metadata field + gwmuxOptions = append(gwmuxOptions, runtime.WithMetadata(auth.GetHTTPRequestCookieToMetadataHandler(authCtx))) + + // In an attempt to be able to selectively enforce whether or not authentication is required, we're going to tag + // the requests that come from the HTTP gateway. See the enforceHttp/Grpc options for more information. + gwmuxOptions = append(gwmuxOptions, runtime.WithMetadata(auth.GetHTTPMetadataTaggingHandler())) + } + + // Create the grpc-gateway server with the options specified + gwmux := runtime.NewServeMux(gwmuxOptions...) + + err := service.RegisterAdminServiceHandlerFromEndpoint(ctx, gwmux, grpcAddress, grpcConnectionOpts) + if err != nil { + return nil, errors.Wrap(err, "error registering admin service") + } + + err = service.RegisterAuthMetadataServiceHandlerFromEndpoint(ctx, gwmux, grpcAddress, grpcConnectionOpts) + if err != nil { + return nil, errors.Wrap(err, "error registering auth service") + } + + err = service.RegisterIdentityServiceHandlerFromEndpoint(ctx, gwmux, grpcAddress, grpcConnectionOpts) + if err != nil { + return nil, errors.Wrap(err, "error registering identity service") + } + + mux.Handle("/", gwmux) + + return mux, nil +} + +func serveGatewayInsecure(ctx context.Context, cfg *config.ServerConfig, authCfg *authConfig.Config) error { + logger.Infof(ctx, "Serving Flyte Admin Insecure") + + // This will parse configuration and create the necessary objects for dealing with auth + var authCtx interfaces.AuthenticationContext + var err error + // This code is here to support authentication without SSL. This setup supports a network topology where + // Envoy does the SSL termination. The final hop is made over localhost only on a trusted machine. + // Warning: Running authentication without SSL in any other topology is a severe security flaw. + // See the auth.Config object for additional settings as well. + if cfg.Security.UseAuth { + sm := secretmanager.NewFileEnvSecretManager(secretmanager.GetConfig()) + var oauth2Provider interfaces.OAuth2Provider + var oauth2ResourceServer interfaces.OAuth2ResourceServer + if authCfg.AppAuth.AuthServerType == authConfig.AuthorizationServerTypeSelf { + oauth2Provider, err = authzserver.NewProvider(ctx, authCfg.AppAuth.SelfAuthServer, sm) + if err != nil { + logger.Errorf(ctx, "Error creating authorization server %s", err) + return err + } + + oauth2ResourceServer = oauth2Provider + } else { + oauth2ResourceServer, err = authzserver.NewOAuth2ResourceServer(ctx, authCfg.AppAuth.ExternalAuthServer, authCfg.UserAuth.OpenID.BaseURL) + if err != nil { + logger.Errorf(ctx, "Error creating resource server %s", err) + return err + } + } + + oauth2MetadataProvider := authzserver.NewService(authCfg) + oidcUserInfoProvider := auth.NewUserInfoProvider() + + authCtx, err = auth.NewAuthenticationContext(ctx, sm, oauth2Provider, oauth2ResourceServer, oauth2MetadataProvider, oidcUserInfoProvider, authCfg) + if err != nil { + logger.Errorf(ctx, "Error creating auth context %s", err) + return err + } + } + + grpcServer, err := newGRPCServer(ctx, cfg, authCtx) + if err != nil { + return errors.Wrap(err, "failed to create GRPC server") + } + + logger.Infof(ctx, "Serving GRPC Traffic on: %s", cfg.GetGrpcHostAddress()) + lis, err := net.Listen("tcp", cfg.GetGrpcHostAddress()) + if err != nil { + return errors.Wrapf(err, "failed to listen on GRPC port: %s", cfg.GetGrpcHostAddress()) + } + + go func() { + err := grpcServer.Serve(lis) + logger.Fatalf(ctx, "Failed to create GRPC Server, Err: ", err) + }() + + logger.Infof(ctx, "Starting HTTP/1 Gateway server on %s", cfg.GetHostAddress()) + grpcOptions := []grpc.DialOption{ + grpc.WithInsecure(), + grpc.WithMaxHeaderListSize(common.MaxResponseStatusBytes), + } + if cfg.GrpcConfig.MaxMessageSizeBytes > 0 { + grpcOptions = append(grpcOptions, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(cfg.GrpcConfig.MaxMessageSizeBytes))) + } + httpServer, err := newHTTPServer(ctx, cfg, authCfg, authCtx, cfg.GetGrpcHostAddress(), grpcOptions...) + if err != nil { + return err + } + + var handler http.Handler + if cfg.Security.AllowCors { + handler = handlers.CORS( + handlers.AllowCredentials(), + handlers.AllowedOrigins(cfg.Security.AllowedOrigins), + handlers.AllowedHeaders(append(defaultCorsHeaders, cfg.Security.AllowedHeaders...)), + handlers.AllowedMethods([]string{"GET", "POST", "DELETE", "HEAD", "PUT", "PATCH"}), + )(httpServer) + } else { + handler = httpServer + } + + err = http.ListenAndServe(cfg.GetHostAddress(), handler) + if err != nil { + return errors.Wrapf(err, "failed to Start HTTP Server") + } + + return nil +} + +// grpcHandlerFunc returns an http.Handler that delegates to grpcServer on incoming gRPC +// connections or otherHandler otherwise. +// See https://github.com/philips/grpc-gateway-example/blob/master/cmd/serve.go for reference +func grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // This is a partial recreation of gRPC's internal checks + if r.ProtoMajor == 2 && strings.Contains(r.Header.Get("Content-Type"), "application/grpc") { + grpcServer.ServeHTTP(w, r) + } else { + otherHandler.ServeHTTP(w, r) + } + }) +} + +func serveGatewaySecure(ctx context.Context, cfg *config.ServerConfig, authCfg *authConfig.Config) error { + certPool, cert, err := GetSslCredentials(ctx, cfg.Security.Ssl.CertificateFile, cfg.Security.Ssl.KeyFile) + if err != nil { + return err + } + // This will parse configuration and create the necessary objects for dealing with auth + var authCtx interfaces.AuthenticationContext + if cfg.Security.UseAuth { + sm := secretmanager.NewFileEnvSecretManager(secretmanager.GetConfig()) + var oauth2Provider interfaces.OAuth2Provider + var oauth2ResourceServer interfaces.OAuth2ResourceServer + if authCfg.AppAuth.AuthServerType == authConfig.AuthorizationServerTypeSelf { + oauth2Provider, err = authzserver.NewProvider(ctx, authCfg.AppAuth.SelfAuthServer, sm) + if err != nil { + logger.Errorf(ctx, "Error creating authorization server %s", err) + return err + } + + oauth2ResourceServer = oauth2Provider + } else { + oauth2ResourceServer, err = authzserver.NewOAuth2ResourceServer(ctx, authCfg.AppAuth.ExternalAuthServer, authCfg.UserAuth.OpenID.BaseURL) + if err != nil { + logger.Errorf(ctx, "Error creating resource server %s", err) + return err + } + } + + oauth2MetadataProvider := authzserver.NewService(authCfg) + oidcUserInfoProvider := auth.NewUserInfoProvider() + + authCtx, err = auth.NewAuthenticationContext(ctx, sm, oauth2Provider, oauth2ResourceServer, oauth2MetadataProvider, oidcUserInfoProvider, authCfg) + if err != nil { + logger.Errorf(ctx, "Error creating auth context %s", err) + return err + } + } + + grpcServer, err := newGRPCServer(ctx, cfg, authCtx, + grpc.Creds(credentials.NewServerTLSFromCert(cert))) + if err != nil { + return errors.Wrap(err, "failed to create GRPC server") + } + + // Whatever certificate is used, pass it along for easier development + dialCreds := credentials.NewTLS(&tls.Config{ + ServerName: cfg.GetHostAddress(), + RootCAs: certPool, + }) + serverOpts := []grpc.DialOption{ + grpc.WithTransportCredentials(dialCreds), + } + if cfg.GrpcConfig.MaxMessageSizeBytes > 0 { + serverOpts = append(serverOpts, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(cfg.GrpcConfig.MaxMessageSizeBytes))) + } + httpServer, err := newHTTPServer(ctx, cfg, authCfg, authCtx, cfg.GetHostAddress(), serverOpts...) + if err != nil { + return err + } + + conn, err := net.Listen("tcp", cfg.GetHostAddress()) + if err != nil { + panic(err) + } + + srv := &http.Server{ + Addr: cfg.GetHostAddress(), + Handler: grpcHandlerFunc(grpcServer, httpServer), + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{*cert}, + NextProtos: []string{"h2"}, + }, + } + + err = srv.Serve(tls.NewListener(conn, srv.TLSConfig)) + + if err != nil { + return errors.Wrapf(err, "failed to Start HTTP/2 Server") + } + return nil +} From 25026261e06a9d7617f064c309ebc1d35b84b71d Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Thu, 10 Mar 2022 23:01:41 -0800 Subject: [PATCH 03/14] Clusterresource controller refactor Signed-off-by: Ketan Umare --- cmd/entrypoints/clusterresource.go | 70 +++++------------------------- pkg/clusterresource/controller.go | 54 ++++++++++++++++++++++- 2 files changed, 64 insertions(+), 60 deletions(-) diff --git a/cmd/entrypoints/clusterresource.go b/cmd/entrypoints/clusterresource.go index 81653d7b4..09016743d 100644 --- a/cmd/entrypoints/clusterresource.go +++ b/cmd/entrypoints/clusterresource.go @@ -4,22 +4,10 @@ import ( "context" errors2 "github.com/pkg/errors" - "github.com/flyteorg/flyteadmin/pkg/repositories/errors" - - "github.com/flyteorg/flyteadmin/pkg/clusterresource/impl" - "github.com/flyteorg/flyteadmin/pkg/clusterresource/interfaces" - execClusterIfaces "github.com/flyteorg/flyteadmin/pkg/executioncluster/interfaces" - "github.com/flyteorg/flyteadmin/pkg/manager/impl/resources" - "github.com/flyteorg/flyteadmin/pkg/repositories" "github.com/flyteorg/flytestdlib/promutils" - "github.com/flyteorg/flyteidl/clients/go/admin" - "github.com/flyteorg/flyteadmin/pkg/clusterresource" - "github.com/flyteorg/flyteadmin/pkg/config" - executioncluster "github.com/flyteorg/flyteadmin/pkg/executioncluster/impl" "github.com/flyteorg/flyteadmin/pkg/runtime" - runtimeInterfaces "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" "github.com/flyteorg/flytestdlib/logger" "github.com/spf13/cobra" @@ -31,58 +19,20 @@ var parentClusterResourceCmd = &cobra.Command{ Short: "This command administers the ClusterResourceController. Please choose a subcommand.", } -func getClusterResourceController(ctx context.Context, scope promutils.Scope, configuration runtimeInterfaces.Configuration) clusterresource.Controller { - initializationErrorCounter := scope.MustNewCounter( - "flyteclient_initialization_error", - "count of errors encountered initializing a flyte client from kube config") - var listTargetsProvider execClusterIfaces.ListTargetsInterface - var err error - if len(configuration.ClusterConfiguration().GetClusterConfigs()) == 0 { - serverConfig := config.GetConfig() - listTargetsProvider, err = executioncluster.NewInCluster(initializationErrorCounter, serverConfig.KubeConfig, serverConfig.Master) - } else { - listTargetsProvider, err = executioncluster.NewListTargets(initializationErrorCounter, executioncluster.NewExecutionTargetProvider(), configuration.ClusterConfiguration()) - } - if err != nil { - panic(err) - } - - var adminDataProvider interfaces.FlyteAdminDataProvider - if configuration.ClusterResourceConfiguration().IsStandaloneDeployment() { - clientSet, err := admin.ClientSetBuilder().WithConfig(admin.GetConfig(ctx)).Build(ctx) - if err != nil { - panic(err) - } - adminDataProvider = impl.NewAdminServiceDataProvider(clientSet.AdminClient()) - } else { - dbConfig := runtime.NewConfigurationProvider().ApplicationConfiguration().GetDbConfig() - logConfig := logger.GetConfig() - - db, err := repositories.GetDB(ctx, dbConfig, logConfig) - if err != nil { - logger.Fatal(ctx, err) - } - dbScope := scope.NewSubScope("db") - - repo := repositories.NewGormRepo( - db, errors.NewPostgresErrorTransformer(dbScope.NewSubScope("errors")), dbScope) - - adminDataProvider = impl.NewDatabaseAdminDataProvider(repo, configuration, resources.NewResourceManager(repo, configuration.ApplicationConfiguration())) - } - - return clusterresource.NewClusterResourceController(adminDataProvider, listTargetsProvider, scope) -} - var controllerRunCmd = &cobra.Command{ Use: "run", Short: "This command will start a cluster resource controller to periodically sync cluster resources", - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() configuration := runtime.NewConfigurationProvider() scope := promutils.NewScope(configuration.ApplicationConfiguration().GetTopLevelConfig().MetricsScope).NewSubScope("clusterresource") - clusterResourceController := getClusterResourceController(ctx, scope, configuration) + clusterResourceController, err := clusterresource.NewClusterResourceControllerFromConfig(ctx, scope, configuration) + if err != nil { + return err + } clusterResourceController.Run() logger.Infof(ctx, "ClusterResourceController started running successfully") + return nil }, } @@ -93,12 +43,16 @@ var controllerSyncCmd = &cobra.Command{ ctx := context.Background() configuration := runtime.NewConfigurationProvider() scope := promutils.NewScope(configuration.ApplicationConfiguration().GetTopLevelConfig().MetricsScope).NewSubScope("clusterresource") - clusterResourceController := getClusterResourceController(ctx, scope, configuration) - err := clusterResourceController.Sync(ctx) + clusterResourceController, err := clusterresource.NewClusterResourceControllerFromConfig(ctx, scope, configuration) + if err != nil { + return err + } + err = clusterResourceController.Sync(ctx) if err != nil { return errors2.Wrap(err, "Failed to sync cluster resources ") } logger.Infof(ctx, "ClusterResourceController synced successfully") + return nil }, } diff --git a/pkg/clusterresource/controller.go b/pkg/clusterresource/controller.go index 677d2894e..7fd4c3f69 100644 --- a/pkg/clusterresource/controller.go +++ b/pkg/clusterresource/controller.go @@ -4,6 +4,13 @@ import ( "context" "encoding/json" "fmt" + impl2 "github.com/flyteorg/flyteadmin/pkg/clusterresource/impl" + "github.com/flyteorg/flyteadmin/pkg/config" + "github.com/flyteorg/flyteadmin/pkg/executioncluster/impl" + "github.com/flyteorg/flyteadmin/pkg/manager/impl/resources" + "github.com/flyteorg/flyteadmin/pkg/repositories" + errors2 "github.com/flyteorg/flyteadmin/pkg/repositories/errors" + admin2 "github.com/flyteorg/flyteidl/clients/go/admin" "io/ioutil" "os" "path" @@ -633,13 +640,56 @@ func newMetrics(scope promutils.Scope) controllerMetrics { } func NewClusterResourceController(adminDataProvider interfaces.FlyteAdminDataProvider, listTargets executionclusterIfaces.ListTargetsInterface, scope promutils.Scope) Controller { - config := runtime.NewConfigurationProvider() + cfg := runtime.NewConfigurationProvider() return &controller{ adminDataProvider: adminDataProvider, - config: config, + config: cfg, listTargets: listTargets, poller: make(chan struct{}), metrics: newMetrics(scope), appliedTemplates: make(map[string]map[string]time.Time), } } + +func NewClusterResourceControllerFromConfig(ctx context.Context, scope promutils.Scope, configuration runtimeInterfaces.Configuration) (Controller, error) { + initializationErrorCounter := scope.MustNewCounter( + "flyteclient_initialization_error", + "count of errors encountered initializing a flyte client from kube config") + var listTargetsProvider executionclusterIfaces.ListTargetsInterface + var err error + if len(configuration.ClusterConfiguration().GetClusterConfigs()) == 0 { + serverConfig := config.GetConfig() + listTargetsProvider, err = impl.NewInCluster(initializationErrorCounter, serverConfig.KubeConfig, serverConfig.Master) + } else { + listTargetsProvider, err = impl.NewListTargets(initializationErrorCounter, impl.NewExecutionTargetProvider(), configuration.ClusterConfiguration()) + } + if err != nil { + return nil, err + } + + var adminDataProvider interfaces.FlyteAdminDataProvider + if configuration.ClusterResourceConfiguration().IsStandaloneDeployment() { + clientSet, err := admin2.ClientSetBuilder().WithConfig(admin2.GetConfig(ctx)).Build(ctx) + if err != nil { + return nil, err + } + adminDataProvider = impl2.NewAdminServiceDataProvider(clientSet.AdminClient()) + } else { + dbConfig := runtime.NewConfigurationProvider().ApplicationConfiguration().GetDbConfig() + logConfig := logger.GetConfig() + + db, err := repositories.GetDB(ctx, dbConfig, logConfig) + if err != nil { + return nil, err + } + dbScope := scope.NewSubScope("db") + + repo := repositories.NewGormRepo( + db, errors2.NewPostgresErrorTransformer(dbScope.NewSubScope("errors")), dbScope) + + adminDataProvider = impl2.NewDatabaseAdminDataProvider(repo, configuration, resources.NewResourceManager(repo, configuration.ApplicationConfiguration())) + } + + return NewClusterResourceController(adminDataProvider, listTargetsProvider, scope), nil +} + From fcada38688ea793e45a5522cd67d813af852860b Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Fri, 11 Mar 2022 12:43:26 -0800 Subject: [PATCH 04/14] lint fix Signed-off-by: Ketan Umare --- cmd/entrypoints/clusterresource.go | 1 + cmd/entrypoints/migrate.go | 1 + cmd/entrypoints/serve.go | 5 +- cmd/scheduler/entrypoints/scheduler.go | 55 +--------------- pkg/clusterresource/controller.go | 16 ++--- pkg/repositories/database.go | 3 +- .../interfaces/application_configuration.go | 5 +- pkg/server/initialize.go | 1 + pkg/server/service.go | 30 ++++----- scheduler/start.go | 62 +++++++++++++++++++ 10 files changed, 95 insertions(+), 84 deletions(-) create mode 100644 scheduler/start.go diff --git a/cmd/entrypoints/clusterresource.go b/cmd/entrypoints/clusterresource.go index 09016743d..4091ce25d 100644 --- a/cmd/entrypoints/clusterresource.go +++ b/cmd/entrypoints/clusterresource.go @@ -2,6 +2,7 @@ package entrypoints import ( "context" + errors2 "github.com/pkg/errors" "github.com/flyteorg/flytestdlib/promutils" diff --git a/cmd/entrypoints/migrate.go b/cmd/entrypoints/migrate.go index 313e1c52b..030a39390 100644 --- a/cmd/entrypoints/migrate.go +++ b/cmd/entrypoints/migrate.go @@ -2,6 +2,7 @@ package entrypoints import ( "context" + "github.com/flyteorg/flyteadmin/pkg/server" "github.com/spf13/cobra" diff --git a/cmd/entrypoints/serve.go b/cmd/entrypoints/serve.go index cad0b70b1..e60fffb43 100644 --- a/cmd/entrypoints/serve.go +++ b/cmd/entrypoints/serve.go @@ -2,13 +2,15 @@ package entrypoints import ( "context" + "github.com/flyteorg/flytestdlib/profutils" + _ "net/http/pprof" // Required to serve application. + "github.com/flyteorg/flyteadmin/pkg/common" "github.com/flyteorg/flyteadmin/pkg/server" "github.com/flyteorg/flytestdlib/logger" "github.com/spf13/cobra" - _ "net/http/pprof" // Required to serve application. runtimeConfig "github.com/flyteorg/flyteadmin/pkg/runtime" "github.com/flyteorg/flytestdlib/contextutils" @@ -45,4 +47,3 @@ func init() { contextutils.ExecIDKey, contextutils.WorkflowIDKey, contextutils.NodeIDKey, contextutils.TaskIDKey, contextutils.TaskTypeKey, common.RuntimeTypeKey, common.RuntimeVersionKey) } - diff --git a/cmd/scheduler/entrypoints/scheduler.go b/cmd/scheduler/entrypoints/scheduler.go index b981a3452..e7ebe9f09 100644 --- a/cmd/scheduler/entrypoints/scheduler.go +++ b/cmd/scheduler/entrypoints/scheduler.go @@ -2,20 +2,13 @@ package entrypoints import ( "context" - "fmt" - "runtime/debug" - - "github.com/flyteorg/flyteadmin/pkg/repositories" - "github.com/flyteorg/flyteadmin/pkg/repositories/errors" "github.com/flyteorg/flyteadmin/pkg/common" "github.com/flyteorg/flyteadmin/pkg/runtime" "github.com/flyteorg/flyteadmin/scheduler" - "github.com/flyteorg/flyteidl/clients/go/admin" "github.com/flyteorg/flytestdlib/contextutils" "github.com/flyteorg/flytestdlib/logger" "github.com/flyteorg/flytestdlib/profutils" - "github.com/flyteorg/flytestdlib/promutils" "github.com/flyteorg/flytestdlib/promutils/labeled" "github.com/spf13/cobra" @@ -27,45 +20,7 @@ var schedulerRunCmd = &cobra.Command{ Short: "This command will start the flyte native scheduler and periodically get new schedules from the db for scheduling", RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() - configuration := runtime.NewConfigurationProvider() - applicationConfiguration := configuration.ApplicationConfiguration().GetTopLevelConfig() - schedulerConfiguration := configuration.ApplicationConfiguration().GetSchedulerConfig() - - // Define the schedulerScope for prometheus metrics - schedulerScope := promutils.NewScope(applicationConfiguration.MetricsScope).NewSubScope("flytescheduler") - schedulerPanics := schedulerScope.MustNewCounter("initialization_panic", - "panics encountered initializing the flyte native scheduler") - - defer func() { - if err := recover(); err != nil { - schedulerPanics.Inc() - logger.Fatalf(ctx, fmt.Sprintf("caught panic: %v [%+v]", err, string(debug.Stack()))) - } - }() - - databaseConfig := configuration.ApplicationConfiguration().GetDbConfig() - logConfig := logger.GetConfig() - - db, err := repositories.GetDB(ctx, databaseConfig, logConfig) - if err != nil { - logger.Fatal(ctx, err) - } - dbScope := schedulerScope.NewSubScope("database") - repo := repositories.NewGormRepo( - db, errors.NewPostgresErrorTransformer(schedulerScope.NewSubScope("errors")), dbScope) - - clientSet, err := admin.ClientSetBuilder().WithConfig(admin.GetConfig(ctx)).Build(ctx) - if err != nil { - logger.Fatalf(ctx, "Flyte native scheduler failed to start due to %v", err) - return err - } - adminServiceClient := clientSet.AdminClient() - - scheduleExecutor := scheduler.NewScheduledExecutor(repo, - configuration.ApplicationConfiguration().GetSchedulerConfig().GetWorkflowExecutorConfig(), schedulerScope, adminServiceClient) - - logger.Info(ctx, "Successfully initialized a native flyte scheduler") - + schedulerConfiguration := runtime.NewConfigurationProvider().ApplicationConfiguration().GetSchedulerConfig() // Serve profiling endpoints. go func() { err := profutils.StartProfilingServerWithDefaultHandlers( @@ -74,13 +29,7 @@ var schedulerRunCmd = &cobra.Command{ logger.Panicf(ctx, "Failed to Start profiling and Metrics server. Error, %v", err) } }() - - err = scheduleExecutor.Run(ctx) - if err != nil { - logger.Fatalf(ctx, "Flyte native scheduler failed to start due to %v", err) - return err - } - return nil + return scheduler.StartScheduler(ctx) }, } diff --git a/pkg/clusterresource/controller.go b/pkg/clusterresource/controller.go index 7fd4c3f69..1fb55e058 100644 --- a/pkg/clusterresource/controller.go +++ b/pkg/clusterresource/controller.go @@ -4,13 +4,6 @@ import ( "context" "encoding/json" "fmt" - impl2 "github.com/flyteorg/flyteadmin/pkg/clusterresource/impl" - "github.com/flyteorg/flyteadmin/pkg/config" - "github.com/flyteorg/flyteadmin/pkg/executioncluster/impl" - "github.com/flyteorg/flyteadmin/pkg/manager/impl/resources" - "github.com/flyteorg/flyteadmin/pkg/repositories" - errors2 "github.com/flyteorg/flyteadmin/pkg/repositories/errors" - admin2 "github.com/flyteorg/flyteidl/clients/go/admin" "io/ioutil" "os" "path" @@ -19,6 +12,14 @@ import ( "strings" "time" + impl2 "github.com/flyteorg/flyteadmin/pkg/clusterresource/impl" + "github.com/flyteorg/flyteadmin/pkg/config" + "github.com/flyteorg/flyteadmin/pkg/executioncluster/impl" + "github.com/flyteorg/flyteadmin/pkg/manager/impl/resources" + "github.com/flyteorg/flyteadmin/pkg/repositories" + errors2 "github.com/flyteorg/flyteadmin/pkg/repositories/errors" + admin2 "github.com/flyteorg/flyteidl/clients/go/admin" + "google.golang.org/grpc/status" "github.com/flyteorg/flyteadmin/pkg/clusterresource/interfaces" @@ -692,4 +693,3 @@ func NewClusterResourceControllerFromConfig(ctx context.Context, scope promutils return NewClusterResourceController(adminDataProvider, listTargetsProvider, scope), nil } - diff --git a/pkg/repositories/database.go b/pkg/repositories/database.go index 92bf7e06d..d80656643 100644 --- a/pkg/repositories/database.go +++ b/pkg/repositories/database.go @@ -3,11 +3,12 @@ package repositories import ( "context" "fmt" - "gorm.io/driver/sqlite" "io/ioutil" "os" "strings" + "gorm.io/driver/sqlite" + runtimeInterfaces "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" "github.com/flyteorg/flytestdlib/logger" "gorm.io/driver/postgres" diff --git a/pkg/runtime/interfaces/application_configuration.go b/pkg/runtime/interfaces/application_configuration.go index 67731aa83..9f2a5d0ab 100644 --- a/pkg/runtime/interfaces/application_configuration.go +++ b/pkg/runtime/interfaces/application_configuration.go @@ -6,7 +6,6 @@ import ( "golang.org/x/time/rate" ) - // DbConfig is used to for initiating the database connection with the store that holds registered // entities (e.g. workflows, tasks, launch plans...) type DbConfig struct { @@ -23,8 +22,8 @@ type DbConfig struct { MaxIdleConnections int `json:"maxIdleConnections" pflag:",maxIdleConnections sets the maximum number of connections in the idle connection pool."` MaxOpenConnections int `json:"maxOpenConnections" pflag:",maxOpenConnections sets the maximum number of open connections to the database."` ConnMaxLifeTime config.Duration `json:"connMaxLifeTime" pflag:",sets the maximum amount of time a connection may be reused"` - PostgresConfig *PostgresConfig `json:"postgres,omitempty"` - SQLiteConfig *SQLiteConfig `json:"sqlite,omitempty"` + PostgresConfig *PostgresConfig `json:"postgres,omitempty"` + SQLiteConfig *SQLiteConfig `json:"sqlite,omitempty"` } // SQLiteConfig can be used to configure diff --git a/pkg/server/initialize.go b/pkg/server/initialize.go index 2352daa37..69c384676 100644 --- a/pkg/server/initialize.go +++ b/pkg/server/initialize.go @@ -3,6 +3,7 @@ package server import ( "context" "fmt" + "github.com/flyteorg/flyteadmin/pkg/repositories" "github.com/flyteorg/flyteadmin/pkg/repositories/config" "github.com/flyteorg/flyteadmin/pkg/runtime" diff --git a/pkg/server/service.go b/pkg/server/service.go index 5c683a67c..b9405a80a 100644 --- a/pkg/server/service.go +++ b/pkg/server/service.go @@ -3,6 +3,10 @@ package server import ( "context" "crypto/tls" + "net" + "net/http" + "strings" + "github.com/flyteorg/flyteadmin/auth" "github.com/flyteorg/flyteadmin/auth/authzserver" authConfig "github.com/flyteorg/flyteadmin/auth/config" @@ -14,9 +18,9 @@ import ( "github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/secretmanager" "github.com/flyteorg/flytestdlib/logger" "github.com/gorilla/handlers" - "github.com/grpc-ecosystem/go-grpc-middleware" - "github.com/grpc-ecosystem/go-grpc-middleware/auth" - "github.com/grpc-ecosystem/go-grpc-prometheus" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/grpc-ecosystem/grpc-gateway/runtime" "github.com/pkg/errors" "google.golang.org/grpc" @@ -26,9 +30,6 @@ import ( "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/reflection" "google.golang.org/grpc/status" - "net" - "net/http" - "strings" ) var defaultCorsHeaders = []string{"Content-Type"} @@ -62,7 +63,7 @@ func blanketAuthorization(ctx context.Context, req interface{}, _ *grpc.UnarySer // Creates a new gRPC Server with all the configuration func newGRPCServer(ctx context.Context, cfg *config.ServerConfig, authCtx interfaces.AuthenticationContext, - opts ...grpc.ServerOption) (*grpc.Server, error) { + opts ...grpc.ServerOption) *grpc.Server { // Not yet implemented for streaming var chainedUnaryInterceptors grpc.UnaryServerInterceptor if cfg.Security.UseAuth { @@ -100,7 +101,7 @@ func newGRPCServer(ctx context.Context, cfg *config.ServerConfig, authCtx interf if cfg.GrpcConfig.ServerReflection || cfg.GrpcServerReflection { reflection.Register(grpcServer) } - return grpcServer, nil + return grpcServer } func GetHandleOpenapiSpec(ctx context.Context) http.HandlerFunc { @@ -218,10 +219,7 @@ func serveGatewayInsecure(ctx context.Context, cfg *config.ServerConfig, authCfg } } - grpcServer, err := newGRPCServer(ctx, cfg, authCtx) - if err != nil { - return errors.Wrap(err, "failed to create GRPC server") - } + grpcServer := newGRPCServer(ctx, cfg, authCtx) logger.Infof(ctx, "Serving GRPC Traffic on: %s", cfg.GetGrpcHostAddress()) lis, err := net.Listen("tcp", cfg.GetGrpcHostAddress()) @@ -319,13 +317,10 @@ func serveGatewaySecure(ctx context.Context, cfg *config.ServerConfig, authCfg * } } - grpcServer, err := newGRPCServer(ctx, cfg, authCtx, - grpc.Creds(credentials.NewServerTLSFromCert(cert))) - if err != nil { - return errors.Wrap(err, "failed to create GRPC server") - } + grpcServer := newGRPCServer(ctx, cfg, authCtx, grpc.Creds(credentials.NewServerTLSFromCert(cert))) // Whatever certificate is used, pass it along for easier development + // #nosec G402 dialCreds := credentials.NewTLS(&tls.Config{ ServerName: cfg.GetHostAddress(), RootCAs: certPool, @@ -350,6 +345,7 @@ func serveGatewaySecure(ctx context.Context, cfg *config.ServerConfig, authCfg * srv := &http.Server{ Addr: cfg.GetHostAddress(), Handler: grpcHandlerFunc(grpcServer, httpServer), + // #nosec G402 TLSConfig: &tls.Config{ Certificates: []tls.Certificate{*cert}, NextProtos: []string{"h2"}, diff --git a/scheduler/start.go b/scheduler/start.go new file mode 100644 index 000000000..d93d09db8 --- /dev/null +++ b/scheduler/start.go @@ -0,0 +1,62 @@ +package scheduler + +import ( + "context" + "fmt" + "runtime/debug" + + "github.com/flyteorg/flyteadmin/pkg/repositories" + "github.com/flyteorg/flyteadmin/pkg/repositories/errors" + "github.com/flyteorg/flyteadmin/pkg/runtime" + "github.com/flyteorg/flyteidl/clients/go/admin" + "github.com/flyteorg/flytestdlib/logger" + "github.com/flyteorg/flytestdlib/promutils" +) + +// StartScheduler creates and starts a new scheduler instance. This is a blocking call and will block the calling go-routine +func StartScheduler(ctx context.Context) error { + configuration := runtime.NewConfigurationProvider() + applicationConfiguration := configuration.ApplicationConfiguration().GetTopLevelConfig() + + // Define the schedulerScope for prometheus metrics + schedulerScope := promutils.NewScope(applicationConfiguration.MetricsScope).NewSubScope("flytescheduler") + schedulerPanics := schedulerScope.MustNewCounter("initialization_panic", + "panics encountered initializing the flyte native scheduler") + + defer func() { + if err := recover(); err != nil { + schedulerPanics.Inc() + logger.Fatalf(ctx, fmt.Sprintf("caught panic: %v [%+v]", err, string(debug.Stack()))) + } + }() + + databaseConfig := configuration.ApplicationConfiguration().GetDbConfig() + logConfig := logger.GetConfig() + + db, err := repositories.GetDB(ctx, databaseConfig, logConfig) + if err != nil { + logger.Fatal(ctx, err) + } + dbScope := schedulerScope.NewSubScope("database") + repo := repositories.NewGormRepo( + db, errors.NewPostgresErrorTransformer(schedulerScope.NewSubScope("errors")), dbScope) + + clientSet, err := admin.ClientSetBuilder().WithConfig(admin.GetConfig(ctx)).Build(ctx) + if err != nil { + logger.Fatalf(ctx, "Flyte native scheduler failed to start due to %v", err) + return err + } + adminServiceClient := clientSet.AdminClient() + + scheduleExecutor := NewScheduledExecutor(repo, + configuration.ApplicationConfiguration().GetSchedulerConfig().GetWorkflowExecutorConfig(), schedulerScope, adminServiceClient) + + logger.Info(ctx, "Successfully initialized a native flyte scheduler") + + err = scheduleExecutor.Run(ctx) + if err != nil { + logger.Fatalf(ctx, "Flyte native scheduler failed to start due to %v", err) + return err + } + return nil +} From 336dc83e76f793f030cca7fecf5d2840eb1f2634 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Fri, 11 Mar 2022 12:58:50 -0800 Subject: [PATCH 05/14] test fix Signed-off-by: Ketan Umare --- pkg/repositories/database_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/repositories/database_test.go b/pkg/repositories/database_test.go index 40e564a71..af8108808 100644 --- a/pkg/repositories/database_test.go +++ b/pkg/repositories/database_test.go @@ -59,7 +59,7 @@ func TestResolvePassword(t *testing.T) { } func TestGetPostgresDsn(t *testing.T) { - pgConfig := runtimeInterfaces.PostgresConfig{ + pgConfig := &runtimeInterfaces.PostgresConfig{ Host: "localhost", Port: 5432, DbName: "postgres", From 6606f454a4808eca82076a847a85a594add45c88 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Fri, 11 Mar 2022 13:47:04 -0800 Subject: [PATCH 06/14] updated Signed-off-by: Ketan Umare --- tests/bootstrap.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bootstrap.go b/tests/bootstrap.go index 57ca36416..44ca517ca 100644 --- a/tests/bootstrap.go +++ b/tests/bootstrap.go @@ -23,7 +23,7 @@ var adminScope = promutils.NewScope("flyteadmin") func getDbConfig() *runtimeInterfaces.DbConfig { return &runtimeInterfaces.DbConfig{ - PostgresConfig: runtimeInterfaces.PostgresConfig{ + PostgresConfig: &runtimeInterfaces.PostgresConfig{ Host: "postgres", Port: 5432, DbName: "postgres", @@ -34,7 +34,7 @@ func getDbConfig() *runtimeInterfaces.DbConfig { func getLocalDbConfig() *runtimeInterfaces.DbConfig { return &runtimeInterfaces.DbConfig{ - PostgresConfig: runtimeInterfaces.PostgresConfig{ + PostgresConfig: &runtimeInterfaces.PostgresConfig{ Host: "localhost", Port: 5432, DbName: "flyteadmin", From 4492d035af95784ddf014eaca7cabfbe7e8b22a9 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Fri, 18 Mar 2022 11:39:49 -0700 Subject: [PATCH 07/14] Updated to handle additional servers Signed-off-by: Ketan Umare --- cmd/entrypoints/serve.go | 2 +- pkg/server/service.go | 23 +++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/cmd/entrypoints/serve.go b/cmd/entrypoints/serve.go index e60fffb43..86ae225bb 100644 --- a/cmd/entrypoints/serve.go +++ b/cmd/entrypoints/serve.go @@ -33,7 +33,7 @@ var serveCmd = &cobra.Command{ } }() - return server.Serve(ctx) + return server.Serve(ctx, nil) }, } diff --git a/pkg/server/service.go b/pkg/server/service.go index 8b985366d..3bd07ff51 100644 --- a/pkg/server/service.go +++ b/pkg/server/service.go @@ -35,15 +35,14 @@ import ( var defaultCorsHeaders = []string{"Content-Type"} // Serve starts a server and blocks the calling goroutine -func Serve(ctx context.Context) error { +func Serve(ctx context.Context, additionalHandlers map[string]func(http.ResponseWriter, *http.Request)) error { serverConfig := config.GetConfig() - authCfg := authConfig.GetConfig() if serverConfig.Security.Secure { - return serveGatewaySecure(ctx, serverConfig, authCfg) + return serveGatewaySecure(ctx, serverConfig, authConfig.GetConfig(), additionalHandlers) } - return serveGatewayInsecure(ctx, serverConfig, authCfg) + return serveGatewayInsecure(ctx, serverConfig, authConfig.GetConfig(), additionalHandlers) } func blanketAuthorization(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) ( @@ -125,11 +124,19 @@ func healthCheckFunc(w http.ResponseWriter, _ *http.Request) { } func newHTTPServer(ctx context.Context, cfg *config.ServerConfig, _ *authConfig.Config, authCtx interfaces.AuthenticationContext, + additionalHandlers map[string]func(http.ResponseWriter, *http.Request), grpcAddress string, grpcConnectionOpts ...grpc.DialOption) (*http.ServeMux, error) { // Register the server that will serve HTTP/REST Traffic mux := http.NewServeMux() + // Add any additional handlers that have been passed in for the main HTTP server + if additionalHandlers != nil { + for p, f := range additionalHandlers { + mux.HandleFunc(p, f) + } + } + // Register healthcheck mux.HandleFunc("/healthcheck", healthCheckFunc) @@ -179,7 +186,7 @@ func newHTTPServer(ctx context.Context, cfg *config.ServerConfig, _ *authConfig. return mux, nil } -func serveGatewayInsecure(ctx context.Context, cfg *config.ServerConfig, authCfg *authConfig.Config) error { +func serveGatewayInsecure(ctx context.Context, cfg *config.ServerConfig, authCfg *authConfig.Config, additionalHandlers map[string]func(http.ResponseWriter, *http.Request)) error { logger.Infof(ctx, "Serving Flyte Admin Insecure") // This will parse configuration and create the necessary objects for dealing with auth @@ -241,7 +248,7 @@ func serveGatewayInsecure(ctx context.Context, cfg *config.ServerConfig, authCfg grpcOptions = append(grpcOptions, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(cfg.GrpcConfig.MaxMessageSizeBytes))) } - httpServer, err := newHTTPServer(ctx, cfg, authCfg, authCtx, cfg.GetGrpcHostAddress(), grpcOptions...) + httpServer, err := newHTTPServer(ctx, cfg, authCfg, authCtx, additionalHandlers, cfg.GetGrpcHostAddress(), grpcOptions...) if err != nil { return err } @@ -280,7 +287,7 @@ func grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Ha }) } -func serveGatewaySecure(ctx context.Context, cfg *config.ServerConfig, authCfg *authConfig.Config) error { +func serveGatewaySecure(ctx context.Context, cfg *config.ServerConfig, authCfg *authConfig.Config, additionalHandlers map[string]func(http.ResponseWriter, *http.Request)) error { certPool, cert, err := GetSslCredentials(ctx, cfg.Security.Ssl.CertificateFile, cfg.Security.Ssl.KeyFile) if err != nil { return err @@ -332,7 +339,7 @@ func serveGatewaySecure(ctx context.Context, cfg *config.ServerConfig, authCfg * serverOpts = append(serverOpts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(cfg.GrpcConfig.MaxMessageSizeBytes))) } - httpServer, err := newHTTPServer(ctx, cfg, authCfg, authCtx, cfg.GetHostAddress(), serverOpts...) + httpServer, err := newHTTPServer(ctx, cfg, authCfg, authCtx, additionalHandlers, cfg.GetHostAddress(), serverOpts...) if err != nil { return err } From 81d3f7c1c746f93dca8e28d7f872c60cd4fa036c Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Fri, 18 Mar 2022 12:31:54 -0700 Subject: [PATCH 08/14] lint fix Signed-off-by: Ketan Umare --- pkg/server/service.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/server/service.go b/pkg/server/service.go index 3bd07ff51..41ce92491 100644 --- a/pkg/server/service.go +++ b/pkg/server/service.go @@ -131,10 +131,8 @@ func newHTTPServer(ctx context.Context, cfg *config.ServerConfig, _ *authConfig. mux := http.NewServeMux() // Add any additional handlers that have been passed in for the main HTTP server - if additionalHandlers != nil { - for p, f := range additionalHandlers { - mux.HandleFunc(p, f) - } + for p, f := range additionalHandlers { + mux.HandleFunc(p, f) } // Register healthcheck From df73aea16c5b1bfc104b5b03e73c1427fed786da Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Fri, 18 Mar 2022 16:46:22 -0700 Subject: [PATCH 09/14] wip Signed-off-by: Katrina Rogan --- flyteadmin_config.yaml | 8 ++------ pkg/repositories/database.go | 1 + pkg/repositories/models/base_model.go | 2 +- pkg/runtime/interfaces/application_configuration.go | 1 + pkg/server/initialize.go | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/flyteadmin_config.yaml b/flyteadmin_config.yaml index 08a70756a..914e83ce6 100644 --- a/flyteadmin_config.yaml +++ b/flyteadmin_config.yaml @@ -61,12 +61,8 @@ flyteadmin: - "metadata" - "admin" database: - postgres: - port: 5432 - username: postgres - host: localhost - dbname: postgres - options: "sslmode=disable" + sqlite: + file: admin.db scheduler: eventScheduler: scheme: local diff --git a/pkg/repositories/database.go b/pkg/repositories/database.go index 74f72a2bb..7a4d3b245 100644 --- a/pkg/repositories/database.go +++ b/pkg/repositories/database.go @@ -98,6 +98,7 @@ func GetDB(ctx context.Context, dbConfig *runtimeInterfaces.DbConfig, logConfig if dbConfig.SQLiteConfig.File == "" { return nil, fmt.Errorf("illegal sqlite database configuration. `file` is a required parameter and should be a path") } + logger.Debugf(ctx, "Opening sqlite db connection") gormDb, err = gorm.Open(sqlite.Open(dbConfig.SQLiteConfig.File), gormConfig) if err != nil { return nil, err diff --git a/pkg/repositories/models/base_model.go b/pkg/repositories/models/base_model.go index bfc5776ce..12a2b3354 100644 --- a/pkg/repositories/models/base_model.go +++ b/pkg/repositories/models/base_model.go @@ -6,7 +6,7 @@ import "time" // This is nearly identical to http://doc.gorm.io/models.html#conventions except that flyteadmin models define their // own primary keys rather than use the ID as the primary key type BaseModel struct { - ID uint `gorm:"index;autoIncrement"` + ID uint `gorm:"index"` CreatedAt time.Time UpdatedAt time.Time DeletedAt *time.Time `gorm:"index"` diff --git a/pkg/runtime/interfaces/application_configuration.go b/pkg/runtime/interfaces/application_configuration.go index 9f2a5d0ab..1babf2229 100644 --- a/pkg/runtime/interfaces/application_configuration.go +++ b/pkg/runtime/interfaces/application_configuration.go @@ -29,6 +29,7 @@ type DbConfig struct { // SQLiteConfig can be used to configure type SQLiteConfig struct { File string `json:"file" pflag:",The path to the file (existing or new) where the DB should be created / stored. If existing, then this will be re-used, else a new will be created"` + Debug bool `json:"debug" pflag:" Whether or not to start the database connection with debug mode enabled."` } // PostgresConfig includes specific config options for opening a connection to a postgres database. diff --git a/pkg/server/initialize.go b/pkg/server/initialize.go index 69c384676..5f923ef25 100644 --- a/pkg/server/initialize.go +++ b/pkg/server/initialize.go @@ -43,7 +43,7 @@ func withDB(ctx context.Context, do func(db *gorm.DB) error) error { // Migrate runs all configured migrations func Migrate(ctx context.Context) error { return withDB(ctx, func(db *gorm.DB) error { - m := gormigrate.New(db, gormigrate.DefaultOptions, config.Migrations) + m := gormigrate.New(db.Debug(), gormigrate.DefaultOptions, config.Migrations) if err := m.Migrate(); err != nil { return fmt.Errorf("database migration failed: %v", err) } From 7e5978476888760231bacef6698dbc24d5fe6052 Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Fri, 18 Mar 2022 16:46:29 -0700 Subject: [PATCH 10/14] sqlite Signed-off-by: Katrina Rogan --- admin.db | Bin 0 -> 282624 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 admin.db diff --git a/admin.db b/admin.db new file mode 100644 index 0000000000000000000000000000000000000000..a6f29704004b17e62a62dd8adc237726cdacf0cf GIT binary patch literal 282624 zcmeI*Z)_XqeZX-YDUp_JiMA8xk~*qRc2w(PvKFm4ikEt>Vl$~4TTUFSjk~6I6nU~) zQzUbD^dDqROUFUGzF1fE)rRdwUkpWuz9`T)D~7$<-gJ92AXu?KK!IXF26P4bvhA=p zd+vCDNgge$&SnW;1+>WLo`3gwe$R9F99{Q~*Dus;L)mDWn`K*B9N9l2Nh6k7Q(3t*Cw1oBY_;lbtzK?7 zD%Z7Ey)5$HYMM7T>djkPtr|`L*tyH4V|DpwXz-U?34FtFDx~%-N<3J zGyX(YUKSlV?RKEv(pqNo%SOf4s?E)Et)Vr_n}&eC9c^EZx1U^SVDnxBdS34~A>KHR z=<3~f9?!~ibJF{#?ecovh_)Dgf5>b7%JNGWN!&MK z@y@Po8hO3la^5?t(bNb_B(L9F#Wrr+L6^NxZoj?vHx1Jgp{#o!t(H;ImGydaU0JsIBuQEk@^ z&A!tT`Q47bc6sIE^5v__tEH>L#6O9k{~mc>_j|3XONTP@t8+}oBG{iumWZ3+{p0bNab{<5OyuiEN{}L^qcc0qgIW2^jMOMtSYa`6x z-&W+9+Q}Wr$`?hWXWd5q%9G}Wf8074Z`@B$D%anMq(M^O-7nS%(V=5QJ0wK?Hmw6& z>x(F|g>I;Km#4Dw%#3vJb$4C{tpsm!UUS}z4AS*SXF57lqpCUUg1J$ycyleUo5p5S zT#_o?87Sh=XxO!lnqhXQQDo}Irrd+iu3|1Dmu8Y?S0JVpgSY$FC$sV?S?Y`h!EcwX z8KGx2}>dn_3WH(hztgyVUes!O?(uy`txNSH&lH%1ZJsEFnSpImMiLu?- zo0VS_9gq_{AWq0_d*A)r4&vD@AWpTfS#@P1BhSf_(PLVxx<&TJ+XyZF1WaBGaN9dj5f)i2qV1BNwNouk<(PUT*zT*&!`?*JbAx z($6j$>87e{V;TAQwDfj={k5u}J2RwmKhpfXqGmVPq;yuEo|fKQbA!!4>i-^%hn%1M z5iG#HOQd%v;obFA4Xa|-TFyPauG}n}mFs15esN(z&5OJ3b)#+t_bf^GHu0r3(weHC z7|qD%rjw$|KWDAzzZ+YVGV<)Kbju~%tX(t9?q(x=mx{54>B@Yqs(fzw^106}U!FgC z>XfQPx}`3rhHYxXTC`&DUTm5^{(|^pdTvtud2sUABe`GXewzE++z)c!$yvFz+(Pc) z)Gwz#nEK(=ccz+CZ%%!BYDQ$@2LS{SKmY**5I_I{1Q0-AhY1`znEL$5g_8?~h35(j ziv{O)Fsno@j3&qogVzIDrD&Fv>VVA`hA?3r9sk7ZY3x(pTLUTiWxmKOgwv>IT760JE>4LYn7Tpat`tsh?lAA*uKDkr~zsQ@LNWIZL@SM}tMzd-}juzI- zl^Ywidj09~)N(hQIHTvgXB1X)bWiG1_h_+j@^oJ%#!aDUJ)QNd7}V0oGpSOfim>iN zx3oOwpTMcY$;C()bEBza-Gc6cdh3a_gI|2Xp%giTXSD}YsX`>(eX?~(at1`mJ9$s~ zaMpx~+=G)Jj^zG1_cyslPR&i^E=~Q1cpC7n-1nz`GWC~JU!Pi^`s`G3Dkn1Wg8%{u zAbK>`LM!O@?zxz)=o-R09 z{l^0C_CNJRdU1Tf{@?ljU;L$ieh@$a0R#|0009ILKmY**5I|rT3pn5Zv;W`41I(!* zfB*srAbdktfB*srAbapRL&=FCfB*srAb!k0Q>($ z8V(?U00IagfB*srAbsqT`ZfLO{-fEgR zHtNk=@l?BP-H4~Ou9vN#wo1KewN3GHy?%BsBfn9QMrw_!aoc*kUb79Y+_szUXD#f# zanoqnmR9UdnZJAenXFtWNbjfP-deXrZ~kKm_Fnc!cLYr1?Y3dru4;D#o0Ur2H2dNp zD*f8!m5a-luPU#Wu8PnKE&Ny*5C+jCHv;l{w?N4~OpzVxOd0zA2xu3TDG^k`vyUXN6(s~b;db>6)o9_T?$fdp$pab` z$?CguRz4@%nH}6tIN74@J$aw@!cqgob>)_cWy?HMc*It zy6vrp(L*1>eDqJ#TLLV zEtcD|#kw6V@5=eo%gbvQt|$w6y;{2_CU>{`ylz#l8`XB*(Cj-cf6!z{U6n6sNje>|narLil)C{vb8zVC?Hgg_)k`!|pxipiswgqBZ zF?hQ__uJHeo|2``SP=Zq?T)2|!RmgVijTE3v3m1$2iZ;S33k!O-I8eIgxiLbBPm|p z(v$JFhUJg9nHbxm1JaA419D;q#0j}=@4J87K|H$!#Hsc*tLA(!HEJE!;!yALCvv-gM}eQ6Bm&w|{aEdVz`D700d`u?W|H zr%Qr5o{`VUNzv|tXhrweL?fG(U!0aY2Z9mt&MO+|e$t-)@$nBurbRdO`~y7^|D{Yu zE>25d>2J=xYWk(JLt64Kug(R_&n_D2rmAaW8Tt6M^mc#!wW^;xGo*4q()_%lW+&M0 zO6jaTJuSVr<_4R8)c-vi4>>>iBUpfYmq_o1zz;&hs+hHwvk}#mn`N_dy==}eE-a{d zakss0)UDvwBk2|;zO+VKQ`Hlr8Ts6FQdIfptQGxtV{1~z*&^R^$u?`(%(A;f4&S9> zY+<@GU#lvgTfThmGs~CfPo6rZDv@rfi>ZwK%4||U!dkRq_&zOm?t7fw`((N}@|g6+ zk<^b<^OOG|{UZHgx;*)*eLvZIYT~~pR>!|T{&eOqGG7||_pxWBFOL3j;_K3Z@n4R= zyywRwABg`R$)C}mN%HH*lQ^7N9`3Kj&blXIY4m3!|N7QaQt?K!YJ|nT-LqC4n0&G` zv6Pipj!T_FF#CF1i>wkoM_%lolRY^iWy6bG*f*whOop>K(O>A<)j=LD5~WUR~`x z_iR=+L?6QYjGo+Dt89vUP4`wUj3FV|ldo|9P9@cp)XT*Ds#sng?|n#SzL1qqi!+%F z!mxK*bnl(QldwM3-{<}^NmIzV*mV}47yG{B(%rp@jXJ^7Q|a&BzgeeLQo~{RdDjm; z`iLsn{)XASr&i&hOQ(7RucwE8{PZNBzJJI4M%>3pJ;kGecjmcai%w6${oe9&UwFYC* zeJpAkws}W9F}ID)mUyBPJWG#07wtaM)cj{9VWwK6)wZ>^S@WOz#BR#PlNq~g_CI8g z4}doKseMnTgopU;?!8dh@5l{~GGOD>_uMBsIegY=+WjYtv-^$`iU+?FC&h!`rRam- z_;To9|9h87Fa1>iL`o_@XmRvv@vb|aBgL$|COT6J7NNnFg;(K$$DiA3r^0%McP^4o z?L4)Rl`n}-b|2q58`goH@K>IJ2N$>6L$|blxgJW})}KAdc%i{Ow6u zxL=$(Wd_B*b8GG17fj@P@9%q>h}_in9653S#v>EUz0UU3AUYArq27CL-n~|SV`e~1 zT0Lct4~s*wDSqe=hr&R6m=PGewr%ek7Kcf=hV^9H>NQMHytNIlywyA6nqI2}Up{C( zkAO{M!w`4o&Ld*?Iv>9-_Om|NHBVLf`~Q#RLkv2F00IagfB*srAbk3g0tg_000IagfB*srAbdA00IagfB*srAbapF zBSk+EKmY**5I_I{1Q0*~0R#}(z5x6G?b}fj0R#|0009ILKmY**5I_Kdog%>h{(q;8 z6#Ya10R#|0009ILKmY**5I|u20(}3!eLG4bfB*srAbVE?~UMv8tS zfB*srAbVE?~UMv8tSfB*sr zAbA#!$m&yN5 zPpAGwx;^q?=G&Q#Olqt%(jI$0`-APbC`vZo%*rpyQb(>es>W@*Y~9dWX7kHN#n!6L z&2p`w)v9i?wfEfR((;v(va))<^roUGWYv{RtKEZ^KCgRK>3O|dnXY#Bzmb*aW~5Hd zZO}BVX4|Y7PAmR<>zvop+Um;dYe9qlVJB<16+dNQOL_fP)4Z`!Z{8BE*URlj<+|3Y zmmA`fUrw{{vxO^S3f{cr_2HTzDar)AqVYwK;>5GS!-Z?5NGyS#F7`SMle)za1OS*dDg;`OXtT9P_* z?obJKtx?_-Ct@^gQJc1D*ky5MWicQF)0g7`Gpu}fJO|byqDUC4og=Sh!o{2};z&sy8wDm+w5XVEK(CM7+ zMOM|YDrT)^*P0Doxmh+V*URSo;=+QO*DYHFeGqZV`O?eFYZtC43q6q*??S$R#mMV! zgQ_YPI{EZ*sWTY_fV%{(T+PUHvSh?kEfJH>QEOlSaJ%W^czmUDV(hD*7x6CE!V_*? zgBCcaE6)1k)-e&UdszN>TRv|vzx{Ju|6Eo+B|0$Hb24F!xebrU8+Im^o34fV;tfX9 zRdp>RpOK{zw+ZX*dd)Vpa@%gYpS^mDZgr)*PhQE&vMk-px#8?4x__LHh3gSF?a2S} zVi*m>n}%t*A=q6tEA^(;Hbw9`E1R-<>58(tcHx51u3a-Mu{662p6)LG-8hafeo4cV z*Zp=>b@H!9&aNCCuBh3@W=rc@YVL=UD7MGBTyK6hE1#Z?#B`YtB=B9(V3iE0HRVCKs?9e77MFun*J*sw6pUKLv zh_=;1ZAY$PvF4Sb&Bsd(=s`~g_2w&C`K;)}Q-k^tPL2>|W@!6ii2-d#4ytdxoRLq= zN_Pe|8y(=Mhc@RgRbltMYvQ$%ST4kAh08^lMRV8R@blrJO?Zp7bL$os5v_F7)y4A} zXABe8jIe}O4Bvm^{Ws5L<=I*3oA0=*M|dQBH|KS#dt-GZJn#`LCPC%?{jGlu3lDhr zsNS_LOph)#J)sre`$eNZO!cQ=n9{mlwu0+tY^4&{%$8B>*xqI0TuKMNmrfF-Hay#tbX0sYz!$tqP_v&HYPG|<@TV+$+8EOqtMoY+2skbfL zFn#)3qt&)8t!>u)6w$V+^(Pyq*)+8qVjlS!yD1g7f8ABzDe52f;@Rt3;>}0c%W%yP z)4XuVhf~76|J7Ahtl*d4C~RRqc$a{nPOEzz_{7La>QJ~abpGH{R(^X)x_j7ND4cNX zzsYtkSAOXFTQmQ5WMGN2@fklbkGoAss?hArQ?t*mC`x0aHy23D^X2PXf1 z^+Hx&6bqg-XhC(xKZJfJe)i5iQg~47#!BCE=O(K+UmbF_2qQ79?A!V&|n7LkK0 z-~T(eM*JXv00IagfB*srAb Date: Fri, 18 Mar 2022 17:39:44 -0700 Subject: [PATCH 11/14] pr into pr Signed-off-by: Katrina Rogan --- admin.db | Bin 282624 -> 274432 bytes pkg/clusterresource/controller.go | 2 +- pkg/repositories/config/migrations.go | 49 ++++++++++-------- pkg/repositories/database.go | 27 ++++++---- pkg/repositories/database_test.go | 4 +- .../gormimpl/resource_repo_test.go | 4 +- pkg/repositories/models/base_model.go | 2 +- pkg/repositories/models/execution.go | 8 +++ pkg/repositories/models/execution_event.go | 8 +++ pkg/repositories/models/launch_plan.go | 10 ++++ pkg/repositories/models/named_entity.go | 7 +++ pkg/repositories/models/node_execution.go | 8 +++ .../models/node_execution_event.go | 8 +++ pkg/repositories/models/project.go | 10 ++++ pkg/repositories/models/resource.go | 12 ++++- pkg/repositories/models/task.go | 10 ++++ pkg/repositories/models/task_execution.go | 8 +++ pkg/repositories/models/workflow.go | 10 ++++ pkg/rpc/adminservice/base.go | 2 +- .../interfaces/application_configuration.go | 4 +- pkg/server/initialize.go | 24 ++++++--- scheduler/start.go | 2 +- 22 files changed, 169 insertions(+), 50 deletions(-) diff --git a/admin.db b/admin.db index a6f29704004b17e62a62dd8adc237726cdacf0cf..eb8f7a14b8fe9d082eb7ecd6c09fcd16c5ce278d 100644 GIT binary patch delta 5336 zcmbtYdvF`Y8P|6!>7k$E35lfGm!Q~@72PSe zk|sh<3>0@5!ZF%vx(}A|w#uNgkZvN;%r!7n$bcR>EcakMb zv7HQ^@t#lnef#Zx@89jc=d$me&w98d`vHwc(@jPP859{KXB%2N9?&#)zN1rSte=pl zrp)@8^%M3Ad)QpVcA8!_^)b`N{mhew_YJ-D+xk`Xle%-dJnf@!zt#h0*_9QfB8B+w zQRIXo{LwU&;2(}6e%1J$1vF>3!-HYTcVI{mMuq;c_gn9QF_3@4X8X zxcp_<{lb?g(6d0baPr4gGb%ilFK268pg`V>D&$5jBL}oqkS9N)ZH9dLvi56C4036% z#{n^id>$3Z$B{|ihRlRzFY0B+Mzd+o;ehcLy#F$kK@R@LWvEQ*`wN*Yy9{ek zZYWD`qIp>`Smi-_t-(rjTPq;eDc@u$lh11n@*A`fvg9lD9(+heUfVvOlnJ=aqqcUcSNd?pWBq9*xahL_t-Rq{OJ1V+v_ z)#4|E=oadLQEp%yWabG|p?sKG)jpO_(;Uaa8KY`T$tMnq>ch~zv!`oUk8M}i#@-&A zSDo|PYQ6peuWiAsijr&b<`LAduQTYm4o*1&{`d{hnnaIFU1ALAsdzb zXUjn+kFgF}YbljqWy_e@DMWK+WpH}C++}G^DR<$Lr5!A#y(DZ(s#)@^F!#xHmZO-R zV=uGMun)08b{E^kn#_MUUowB!eAL``!Ij0Dqooy8SF^LZnRmK)my1sZ1y2-{gw^HL zK0Aayi=-D=3~&_u~Bnqotf`9;ade zR~t`M7luS$T}oD#6sQLz)Hc4wrRcLrox*&oJua5mqmA$P5#uRF4T}YNlsisot>#^= z@g1c|P~dW@wgiE1ZciyJUihTULZJ$?Tc&Nqr_Zskv6zjpd)N+^GylWqFRzA*gCuz`M_Zq>c6o6vqr?WT(1+h8OM?mydA z$i!N!X-=qzn1H9=BW=R)Ce%P3o(iE{9DD~@yyH#iMKQ*KJLjMsY8O%V&5jPKX|BE= zzA56FIj8~$9)1@}SuwL@K89a*$r?mWpV z;W;8mz5$=)^V$v!4Idy&g1!)`TAqcXOlH)i^u)7L_UiNTFaHP|QOsh;_0zB(Rw;e7 z3inO}gCCs+hrZLM=lXcHcbJh8Kk+gYnX1Nw3Yz12ICB!Op8*F{;tyVi>SS%6z~^3p z)71afK&iHPb2R1NKqvi>`l=nYM(;{%jwZ|4jkMHzsq6JEdA-%OZ10jUSx!(b6u}v# z_)otFP9Cz})ynG6YVtJA8up^$6n&M=W4@!`se4>!)lL|1rm_Aa`ZM)AYCTjLbFGhS zCJAGApjYkO=fkK8(_kMzO40@drU< zhjz>yMM1a)pE!!jVJ&{_DEbdo*@JB&>VY;T-fQtgB5Fl3Xu@-|um+k5WGKKtn*uW? znZe@wX9>D_Zw45OX~do>l3K0>j&&&3!sD7*Xu+So2f6ZjlQzEaS8 z*VYDSw&K@C)C6w)xrpjhro5KvwNgalyT3qCYz-(C zv@f&Tnd%Z+Z9Z_jitY>x=x!5h_^BovB#rf$BOJ{ns$Omn_&i0xB-P2i0ufzpsbp=wVJI(qls*vH1P0|XOZ(;aYv6lIOqxT|gk^Z*)(9 zBA{}I9C4+O)K#}_)ZAjj7)`Hzv+ zf$$L_OkEC;3W*-!ZYBHF#rLU;Ad8MvlgOxgkq|%gkI6NHVT8`) z!?6hIdt^m398(*{!T~Q%k!GjrcqmLfp04AJoHIpfQ2TMM>b8A<48z5<$*J{K^azo0Ht$I_3g6|JyVEa+9F%T(<~u#(Rd&ZWMKU6z@w$qo}=_Y2iei zs1_#lNQQ;+r%IF!Mp2oT$T2JvMP%W_*G3VUwO<=WMiKoWGdfJEo|N!XO$4>vMp*6F z>20lP34h(*_B6Stf*Zk5+NQ4r&q(H=@8jB8XkG!InVP;D%4u?rA8J<{{{&=Q&v`tO za3q+rV*IyFxvDG7iT}tfdyAWWt%80uS;|9WC&Na@;&4lb8u`3@n|xl9@U*FaS!Z|9 Qb|Bn0U4PYvcwF252cqRg|sTM(@r!srcQrmbjHrq&RAz0XEHUWt&_gp<4%%x znoj@dI5DquYlfs(c_9Mbp|0Rych{i5ComA&vd!Pu z69|R<-CYltc=$4;v1@|*lBkQ_U7L!(p|hhx6^0A=RAN60=}fm)g3j8kPsu~i&ysYn z7Y@|1xuT2Y+QmH7qOc8@{{*tw9dVp^BT|h$eO;=km$_kp{ZUFy>h9XzyYtztJ-xfb zvzX16{RrvF?O2w(DZ$=Uw8_xDGlhL9l|d?PYJp_dVVegjEZJVjUbj_)lijfSm}JwK z+urDDa!9({4f{L$1Dkt-iGvbO+4N9gX6xbdl{|xpA!q0|PyfFn$?0uV)3n^^1Qp7;XXTwUqt5p_tuMWY4f6e_Q zX#B<#S726koD;jLxthM^P=-y?bshF=rtRh{^I>OANm(;&zLwMl#8ZoSIV7j^o$gZhghH5r*Ib7MRt)gD)f*lEC`gZnyHMphU2sFZ*P~eICJN_bW2=J2ba>2gU z`1LoT+R*$&bjnc&j-2_FoG!0Q&lL3@lbqgYvhR`_z2|ape2RLLNx24RQWc8%HNgJn z@?H6fZFFk+Xrt47`c|lNi6sqoTeKeSXO3EJ zL97x!ll`l#S@GPTIb_C1rJL_qjjg z@Oj9eE-G3NePK}vvuZJSD8nNLrPzu%yBU3pdFX zsTbiEf0GZW3zObe=E;ANm)U=4&lbNcK4bmRS^yh_6Y>j?Y`<-9u}xa8TCTEA=TT9( z4ti%PYtk~X?=<8}wZe7F?>E+(jJHmMjLJDs@Z<#OLS;m*#al&u9W221AH#C7zF3UQ znp++#~#4#Hmd=C~t2|@v%h3R9Ef``t6h-9E02gV@1W>=x4FDQdZa-!D+2ZLJ% z`n!8W{@|`)$m}_vXBC_0RXp4fYl)xVHrJO zU#pVz(o*}PLG$uUqF;klt<(t z$|Z6WWfNu5?D;j#p2qn2>6@=w-4@+4V3DuLugDeltM>J_-`Lhmx3z;>9)H2{JM|OZ z#T|x0%T4i!xK-q=5zBzJMtXy)ubvC7B{&e`BhHn$XdFDS7&nbWAFRM%jYAGRhIhx| zW4qiL5BJ?DxQ#SxIz?`5!tzs4WGn-xASdRTyybW+3eC`HyxX7+t$)&t!Ty!87#MQ6 zd6IakMnjdd-f+y_v_XoH4`72`fnu})IWH#09ZIX^V zEGxW^@&D&Q+|;yuVgr>u|?F&J@6a`-h)|cH-YHsE*H*@ofY^cq?vH}E6qnQlftYRyYC_ex@w7m&6UCf zYZproQ3-x?lu~!>9Y{4R==23>=LAa44m>ml`S^*4I#eP3V~}R(jRiCEbO-+QG^~UM zbe@5Ou*@h?7oI)?rShV#cEb1`!uY0*c;Ye$c;X_+P!KbP7&-8Sai?-R-gH3r`%nyK V)ijt@Q%~yui>hh#jZWp#_n+=t)N=p; diff --git a/pkg/clusterresource/controller.go b/pkg/clusterresource/controller.go index 1fb55e058..b53b43339 100644 --- a/pkg/clusterresource/controller.go +++ b/pkg/clusterresource/controller.go @@ -679,7 +679,7 @@ func NewClusterResourceControllerFromConfig(ctx context.Context, scope promutils dbConfig := runtime.NewConfigurationProvider().ApplicationConfiguration().GetDbConfig() logConfig := logger.GetConfig() - db, err := repositories.GetDB(ctx, dbConfig, logConfig) + db, _, err := repositories.GetDB(ctx, dbConfig, logConfig) if err != nil { return nil, err } diff --git a/pkg/repositories/config/migrations.go b/pkg/repositories/config/migrations.go index 0fba3bb03..65a69473e 100644 --- a/pkg/repositories/config/migrations.go +++ b/pkg/repositories/config/migrations.go @@ -200,16 +200,6 @@ var Migrations = []*gormigrate.Migration{ return tx.Exec("UPDATE named_entity_metadata set state = NULL").Error }, }, - // Modify the workflows table, if necessary - { - ID: "2020-04-03-workflow-state", - Migrate: func(tx *gorm.DB) error { - return tx.Exec("ALTER TABLE workflows DROP COLUMN IF EXISTS state").Error - }, - Rollback: func(tx *gorm.DB) error { - return tx.Exec("ALTER TABLE workflows ADD COLUMN IF NOT EXISTS state integer;").Error - }, - }, // Modify the executions & node_execution table, if necessary { ID: "2020-04-29-executions", @@ -340,6 +330,32 @@ var Migrations = []*gormigrate.Migration{ }, }, + // Add state to execution model. + { + ID: "2022-01-11-execution-state", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Execution{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Table("execution").Migrator().DropColumn(&models.Execution{}, "state") + }, + }, +} + +// PostgresMigrations adds onto the defacto migrations. These migrations are exclusive to postgres, because: +// Databases like Sqlite don't support dropping columns. +// Databases like Sqlite don't use rigid column types, so modifying the id column type is unsupported +var PostgresMigrations = append(Migrations, []*gormigrate.Migration{ + // Modify the workflows table, if necessary + { + ID: "2020-04-03-workflow-state", + Migrate: func(tx *gorm.DB) error { + return tx.Exec("ALTER TABLE workflows DROP COLUMN IF EXISTS state").Error + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec("ALTER TABLE workflows ADD COLUMN IF NOT EXISTS state integer;").Error + }, + }, // For any new table, Please use the following pattern due to a bug // in the postgres gorm layer https://github.com/go-gorm/postgres/issues/65 { @@ -359,18 +375,7 @@ var Migrations = []*gormigrate.Migration{ return alterTableColumnType(db, "id", "int") }, }, - - // Add state to execution model. - { - ID: "2022-01-11-execution-state", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Execution{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Table("execution").Migrator().DropColumn(&models.Execution{}, "state") - }, - }, -} +}...) func alterTableColumnType(db *sql.DB, columnName, columnType string) error { diff --git a/pkg/repositories/database.go b/pkg/repositories/database.go index 7a4d3b245..a12fec265 100644 --- a/pkg/repositories/database.go +++ b/pkg/repositories/database.go @@ -22,6 +22,13 @@ import ( const pqInvalidDBCode = "3D000" const defaultDB = "postgres" +type DatabaseType int64 + +const ( + DatabaseTypePostgres DatabaseType = iota + DatabaseTypeSqlite +) + // getGormLogLevel converts between the flytestdlib configured log level to the equivalent gorm log level. func getGormLogLevel(ctx context.Context, logConfig *logger.Config) gormLogger.LogLevel { if logConfig == nil { @@ -80,8 +87,7 @@ func getPostgresDsn(ctx context.Context, pgConfig *runtimeInterfaces.PostgresCon // GetDB uses the dbConfig to create gorm DB object. If the db doesn't exist for the dbConfig then a new one is created // using the default db for the provider. eg : postgres has default dbName as postgres -func GetDB(ctx context.Context, dbConfig *runtimeInterfaces.DbConfig, logConfig *logger.Config) ( - *gorm.DB, error) { +func GetDB(ctx context.Context, dbConfig *runtimeInterfaces.DbConfig, logConfig *logger.Config) (*gorm.DB, DatabaseType, error) { if dbConfig == nil { panic("Cannot initialize database repository from empty db config") } @@ -92,23 +98,25 @@ func GetDB(ctx context.Context, dbConfig *runtimeInterfaces.DbConfig, logConfig var gormDb *gorm.DB var err error + var databaseType DatabaseType switch { case dbConfig.SQLiteConfig != nil: if dbConfig.SQLiteConfig.File == "" { - return nil, fmt.Errorf("illegal sqlite database configuration. `file` is a required parameter and should be a path") + return nil, databaseType, fmt.Errorf("illegal sqlite database configuration. `file` is a required parameter and should be a path") } logger.Debugf(ctx, "Opening sqlite db connection") gormDb, err = gorm.Open(sqlite.Open(dbConfig.SQLiteConfig.File), gormConfig) if err != nil { - return nil, err + return nil, databaseType, err } + databaseType = DatabaseTypeSqlite case dbConfig.PostgresConfig != nil && (len(dbConfig.PostgresConfig.Host) > 0 || len(dbConfig.PostgresConfig.User) > 0 || len(dbConfig.PostgresConfig.DbName) > 0): gormDb, err = createPostgresDbIfNotExists(ctx, gormConfig, dbConfig.PostgresConfig) if err != nil { - return nil, err + return nil, databaseType, err } - + databaseType = DatabaseTypePostgres case len(dbConfig.DeprecatedHost) > 0 || len(dbConfig.DeprecatedUser) > 0 || len(dbConfig.DeprecatedDbName) > 0: pgConfig := &runtimeInterfaces.PostgresConfig{ Host: dbConfig.DeprecatedHost, @@ -122,14 +130,15 @@ func GetDB(ctx context.Context, dbConfig *runtimeInterfaces.DbConfig, logConfig } gormDb, err = createPostgresDbIfNotExists(ctx, gormConfig, pgConfig) if err != nil { - return nil, err + return nil, databaseType, err } + databaseType = DatabaseTypePostgres default: - return nil, fmt.Errorf("unrecognized database config, %v. Supported only postgres and sqlite", dbConfig) + return nil, databaseType, fmt.Errorf("unrecognized database config, %v. Supported only postgres and sqlite", dbConfig) } // Setup connection pool settings - return gormDb, setupDbConnectionPool(gormDb, dbConfig) + return gormDb, databaseType, setupDbConnectionPool(gormDb, dbConfig) } // Creates DB if it doesn't exist for the passed in config diff --git a/pkg/repositories/database_test.go b/pkg/repositories/database_test.go index af8108808..27b53aaac 100644 --- a/pkg/repositories/database_test.go +++ b/pkg/repositories/database_test.go @@ -149,13 +149,13 @@ func TestGetDB(t *testing.T) { ctx := context.TODO() t.Run("missing DB Config", func(t *testing.T) { - _, err := GetDB(ctx, &runtimeInterfaces.DbConfig{}, &logger.Config{}) + _, _, err := GetDB(ctx, &runtimeInterfaces.DbConfig{}, &logger.Config{}) assert.Error(t, err) }) t.Run("sqlite config", func(t *testing.T) { dbFile := path.Join(t.TempDir(), "admin.db") - db, err := GetDB(ctx, &runtimeInterfaces.DbConfig{ + db, _, err := GetDB(ctx, &runtimeInterfaces.DbConfig{ SQLiteConfig: &runtimeInterfaces.SQLiteConfig{ File: dbFile, }, diff --git a/pkg/repositories/gormimpl/resource_repo_test.go b/pkg/repositories/gormimpl/resource_repo_test.go index 00ecb7d41..5c2753580 100644 --- a/pkg/repositories/gormimpl/resource_repo_test.go +++ b/pkg/repositories/gormimpl/resource_repo_test.go @@ -22,7 +22,7 @@ func TestCreateWorkflowAttributes(t *testing.T) { query := GlobalMock.NewMock() GlobalMock.Logging = true query.WithQuery( - `INSERT INTO "resources" ("created_at","updated_at","deleted_at","project","domain","workflow","launch_plan","resource_type","priority","attributes") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING "id"`) + `INSERT INTO "resources" ("created_at","updated_at","deleted_at","project","domain","workflow","launch_plan","resource_type","priority","attributes","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING "id"`) err := resourceRepo.CreateOrUpdate(context.Background(), models.Resource{ Project: "project", @@ -65,7 +65,7 @@ func TestUpdateWorkflowAttributes_WithExisting(t *testing.T) { mockSaveQuery := GlobalMock.NewMock() mockSaveQuery.WithQuery( - `INSERT INTO "resources" ("created_at","updated_at","deleted_at","project","domain","workflow","launch_plan","resource_type","priority","attributes") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`) + `INSERT INTO "resources" ("created_at","updated_at","deleted_at","project","domain","workflow","launch_plan","resource_type","priority","attributes","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)`) err := resourceRepo.CreateOrUpdate(context.Background(), models.Resource{ ResourceType: resourceType.String(), diff --git a/pkg/repositories/models/base_model.go b/pkg/repositories/models/base_model.go index 12a2b3354..8660772b4 100644 --- a/pkg/repositories/models/base_model.go +++ b/pkg/repositories/models/base_model.go @@ -6,7 +6,7 @@ import "time" // This is nearly identical to http://doc.gorm.io/models.html#conventions except that flyteadmin models define their // own primary keys rather than use the ID as the primary key type BaseModel struct { - ID uint `gorm:"index"` + ID uint `gorm:"type:uuid"` CreatedAt time.Time UpdatedAt time.Time DeletedAt *time.Time `gorm:"index"` diff --git a/pkg/repositories/models/execution.go b/pkg/repositories/models/execution.go index fa429226b..5758531ac 100644 --- a/pkg/repositories/models/execution.go +++ b/pkg/repositories/models/execution.go @@ -3,6 +3,9 @@ package models import ( "time" + "github.com/google/uuid" + "gorm.io/gorm" + "github.com/flyteorg/flytestdlib/storage" ) @@ -59,3 +62,8 @@ type Execution struct { // GORM doesn't save the zero value for ints, so we use a pointer for the State field State *int32 `gorm:"index;default:0"` } + +func (e *Execution) BeforeCreate(tx *gorm.DB) error { + e.ID = uint(uuid.New().ID()) + return nil +} diff --git a/pkg/repositories/models/execution_event.go b/pkg/repositories/models/execution_event.go index b6e18decc..ef4fd3990 100644 --- a/pkg/repositories/models/execution_event.go +++ b/pkg/repositories/models/execution_event.go @@ -2,6 +2,9 @@ package models import ( "time" + + "github.com/google/uuid" + "gorm.io/gorm" ) type ExecutionEvent struct { @@ -11,3 +14,8 @@ type ExecutionEvent struct { OccurredAt time.Time Phase string `gorm:"primary_key"` } + +func (e *ExecutionEvent) BeforeCreate(tx *gorm.DB) error { + e.ID = uint(uuid.New().ID()) + return nil +} diff --git a/pkg/repositories/models/launch_plan.go b/pkg/repositories/models/launch_plan.go index 2b2787518..5a411287a 100644 --- a/pkg/repositories/models/launch_plan.go +++ b/pkg/repositories/models/launch_plan.go @@ -1,5 +1,10 @@ package models +import ( + "github.com/google/uuid" + "gorm.io/gorm" +) + // Launch plan primary key type LaunchPlanKey struct { Project string `gorm:"primary_key;index:lp_project_domain_name_idx,lp_project_domain_idx" valid:"length(0|255)"` @@ -32,3 +37,8 @@ type LaunchPlan struct { Digest []byte ScheduleType LaunchPlanScheduleType } + +func (l *LaunchPlan) BeforeCreate(tx *gorm.DB) error { + l.ID = uint(uuid.New().ID()) + return nil +} diff --git a/pkg/repositories/models/named_entity.go b/pkg/repositories/models/named_entity.go index 966676d0c..c73c37473 100644 --- a/pkg/repositories/models/named_entity.go +++ b/pkg/repositories/models/named_entity.go @@ -2,6 +2,8 @@ package models import ( "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + "github.com/google/uuid" + "gorm.io/gorm" ) // NamedEntityMetadata primary key @@ -40,3 +42,8 @@ type NamedEntity struct { NamedEntityKey NamedEntityMetadataFields } + +func (n *NamedEntityMetadata) BeforeCreate(tx *gorm.DB) error { + n.ID = uint(uuid.New().ID()) + return nil +} diff --git a/pkg/repositories/models/node_execution.go b/pkg/repositories/models/node_execution.go index cb0281f6d..f1f9dab08 100644 --- a/pkg/repositories/models/node_execution.go +++ b/pkg/repositories/models/node_execution.go @@ -2,6 +2,9 @@ package models import ( "time" + + "github.com/google/uuid" + "gorm.io/gorm" ) // IMPORTANT: If you update the model below, be sure to double check model definitions in @@ -49,3 +52,8 @@ type NodeExecution struct { // In the case of dynamic workflow nodes, the remote closure is uploaded to the path specified here. DynamicWorkflowRemoteClosureReference string } + +func (n *NodeExecution) BeforeCreate(tx *gorm.DB) error { + n.ID = uint(uuid.New().ID()) + return nil +} diff --git a/pkg/repositories/models/node_execution_event.go b/pkg/repositories/models/node_execution_event.go index cd362c8ff..2ee001b4a 100644 --- a/pkg/repositories/models/node_execution_event.go +++ b/pkg/repositories/models/node_execution_event.go @@ -2,6 +2,9 @@ package models import ( "time" + + "github.com/google/uuid" + "gorm.io/gorm" ) type NodeExecutionEvent struct { @@ -11,3 +14,8 @@ type NodeExecutionEvent struct { OccurredAt time.Time Phase string `gorm:"primary_key"` } + +func (n *NodeExecutionEvent) BeforeCreate(tx *gorm.DB) error { + n.ID = uint(uuid.New().ID()) + return nil +} diff --git a/pkg/repositories/models/project.go b/pkg/repositories/models/project.go index a5feedb27..a35985eb3 100644 --- a/pkg/repositories/models/project.go +++ b/pkg/repositories/models/project.go @@ -1,5 +1,10 @@ package models +import ( + "github.com/google/uuid" + "gorm.io/gorm" +) + type Project struct { BaseModel Identifier string `gorm:"primary_key"` @@ -9,3 +14,8 @@ type Project struct { // GORM doesn't save the zero value for ints, so we use a pointer for the State field State *int32 `gorm:"default:0;index"` } + +func (p *Project) BeforeCreate(tx *gorm.DB) error { + p.ID = uint(uuid.New().ID()) + return nil +} diff --git a/pkg/repositories/models/resource.go b/pkg/repositories/models/resource.go index 02da2fbe3..8d274cf37 100644 --- a/pkg/repositories/models/resource.go +++ b/pkg/repositories/models/resource.go @@ -1,6 +1,11 @@ package models -import "time" +import ( + "time" + + "github.com/google/uuid" + "gorm.io/gorm" +) type ResourcePriority int32 @@ -27,3 +32,8 @@ type Resource struct { // Serialized flyteidl.admin.MatchingAttributes. Attributes []byte } + +func (r *Resource) BeforeCreate(tx *gorm.DB) error { + r.ID = int64(uuid.New().ID()) + return nil +} diff --git a/pkg/repositories/models/task.go b/pkg/repositories/models/task.go index 093bd29ec..09de5e12d 100644 --- a/pkg/repositories/models/task.go +++ b/pkg/repositories/models/task.go @@ -1,5 +1,10 @@ package models +import ( + "github.com/google/uuid" + "gorm.io/gorm" +) + // IMPORTANT: If you update the model below, be sure to double check model definitions in // pkg/repositories/config/migration_models.go @@ -21,3 +26,8 @@ type Task struct { // Task type (also stored in the closure put promoted as a column for filtering). Type string `valid:"length(0|255)"` } + +func (t *Task) BeforeCreate(tx *gorm.DB) error { + t.ID = uint(uuid.New().ID()) + return nil +} diff --git a/pkg/repositories/models/task_execution.go b/pkg/repositories/models/task_execution.go index d1f8ad243..271504812 100644 --- a/pkg/repositories/models/task_execution.go +++ b/pkg/repositories/models/task_execution.go @@ -2,6 +2,9 @@ package models import ( "time" + + "github.com/google/uuid" + "gorm.io/gorm" ) // IMPORTANT: If you update the model below, be sure to double check model definitions in @@ -37,3 +40,8 @@ type TaskExecution struct { // The child node executions (if any) launched by this task execution. ChildNodeExecution []NodeExecution `gorm:"foreignkey:ParentTaskExecutionID;references:ID"` } + +func (t *TaskExecution) BeforeCreate(tx *gorm.DB) error { + t.ID = uint(uuid.New().ID()) + return nil +} diff --git a/pkg/repositories/models/workflow.go b/pkg/repositories/models/workflow.go index 5f50379b1..5d299e4ca 100644 --- a/pkg/repositories/models/workflow.go +++ b/pkg/repositories/models/workflow.go @@ -1,5 +1,10 @@ package models +import ( + "github.com/google/uuid" + "gorm.io/gorm" +) + // Workflow primary key type WorkflowKey struct { Project string `gorm:"primary_key;index:workflow_project_domain_name_idx;index:workflow_project_domain_idx" valid:"length(0|255)"` @@ -17,3 +22,8 @@ type Workflow struct { // Hash of the compiled workflow closure Digest []byte } + +func (w *Workflow) BeforeCreate(tx *gorm.DB) error { + w.ID = uint(uuid.New().ID()) + return nil +} diff --git a/pkg/rpc/adminservice/base.go b/pkg/rpc/adminservice/base.go index f3f740060..c7e6584e5 100644 --- a/pkg/rpc/adminservice/base.go +++ b/pkg/rpc/adminservice/base.go @@ -75,7 +75,7 @@ func NewAdminServer(ctx context.Context, kubeConfig, master string) *AdminServic databaseConfig := configuration.ApplicationConfiguration().GetDbConfig() logConfig := logger.GetConfig() - db, err := repositories.GetDB(ctx, databaseConfig, logConfig) + db, _, err := repositories.GetDB(ctx, databaseConfig, logConfig) if err != nil { logger.Fatal(ctx, err) } diff --git a/pkg/runtime/interfaces/application_configuration.go b/pkg/runtime/interfaces/application_configuration.go index 1babf2229..96f5a7ad5 100644 --- a/pkg/runtime/interfaces/application_configuration.go +++ b/pkg/runtime/interfaces/application_configuration.go @@ -28,8 +28,8 @@ type DbConfig struct { // SQLiteConfig can be used to configure type SQLiteConfig struct { - File string `json:"file" pflag:",The path to the file (existing or new) where the DB should be created / stored. If existing, then this will be re-used, else a new will be created"` - Debug bool `json:"debug" pflag:" Whether or not to start the database connection with debug mode enabled."` + File string `json:"file" pflag:",The path to the file (existing or new) where the DB should be created / stored. If existing, then this will be re-used, else a new will be created"` + Debug bool `json:"debug" pflag:" Whether or not to start the database connection with debug mode enabled."` } // PostgresConfig includes specific config options for opening a connection to a postgres database. diff --git a/pkg/server/initialize.go b/pkg/server/initialize.go index 5f923ef25..7b4c61fea 100644 --- a/pkg/server/initialize.go +++ b/pkg/server/initialize.go @@ -12,12 +12,12 @@ import ( "gorm.io/gorm" ) -func withDB(ctx context.Context, do func(db *gorm.DB) error) error { +func withDB(ctx context.Context, do func(db *gorm.DB, dbType repositories.DatabaseType) error) error { configuration := runtime.NewConfigurationProvider() databaseConfig := configuration.ApplicationConfiguration().GetDbConfig() logConfig := logger.GetConfig() - db, err := repositories.GetDB(ctx, databaseConfig, logConfig) + db, dbType, err := repositories.GetDB(ctx, databaseConfig, logConfig) if err != nil { logger.Fatal(ctx, err) } @@ -37,13 +37,17 @@ func withDB(ctx context.Context, do func(db *gorm.DB) error) error { return err } - return do(db) + return do(db, dbType) } // Migrate runs all configured migrations func Migrate(ctx context.Context) error { - return withDB(ctx, func(db *gorm.DB) error { - m := gormigrate.New(db.Debug(), gormigrate.DefaultOptions, config.Migrations) + return withDB(ctx, func(db *gorm.DB, dbType repositories.DatabaseType) error { + migrations := config.Migrations + if dbType == repositories.DatabaseTypePostgres { + migrations = config.PostgresMigrations + } + m := gormigrate.New(db.Debug(), gormigrate.DefaultOptions, migrations) if err := m.Migrate(); err != nil { return fmt.Errorf("database migration failed: %v", err) } @@ -54,8 +58,12 @@ func Migrate(ctx context.Context) error { // Rollback rolls back the last migration func Rollback(ctx context.Context) error { - return withDB(ctx, func(db *gorm.DB) error { - m := gormigrate.New(db, gormigrate.DefaultOptions, config.Migrations) + return withDB(ctx, func(db *gorm.DB, dbType repositories.DatabaseType) error { + migrations := config.Migrations + if dbType == repositories.DatabaseTypePostgres { + migrations = config.PostgresMigrations + } + m := gormigrate.New(db, gormigrate.DefaultOptions, migrations) err := m.RollbackLast() if err != nil { return fmt.Errorf("could not rollback latest migration: %v", err) @@ -67,7 +75,7 @@ func Rollback(ctx context.Context) error { // SeedProjects creates a set of given projects in the DB func SeedProjects(ctx context.Context, projects []string) error { - return withDB(ctx, func(db *gorm.DB) error { + return withDB(ctx, func(db *gorm.DB, _ repositories.DatabaseType) error { if err := config.SeedProjects(db, projects); err != nil { return fmt.Errorf("could not add projects to database with err: %v", err) } diff --git a/scheduler/start.go b/scheduler/start.go index d93d09db8..77c9f79f3 100644 --- a/scheduler/start.go +++ b/scheduler/start.go @@ -33,7 +33,7 @@ func StartScheduler(ctx context.Context) error { databaseConfig := configuration.ApplicationConfiguration().GetDbConfig() logConfig := logger.GetConfig() - db, err := repositories.GetDB(ctx, databaseConfig, logConfig) + db, _, err := repositories.GetDB(ctx, databaseConfig, logConfig) if err != nil { logger.Fatal(ctx, err) } From da0805d0ed300e15c41c51fba106d09956e16565 Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Tue, 22 Mar 2022 16:27:40 -0700 Subject: [PATCH 12/14] Refactor Signed-off-by: Katrina Rogan --- admin.db | Bin 274432 -> 274432 bytes pkg/repositories/config/migration_models.go | 13 +- pkg/repositories/config/migrations.go | 542 ++++++++++-------- pkg/repositories/models/base_model.go | 2 +- pkg/repositories/models/execution.go | 8 - pkg/repositories/models/execution_event.go | 8 - pkg/repositories/models/launch_plan.go | 10 - pkg/repositories/models/named_entity.go | 7 - pkg/repositories/models/node_execution.go | 5 +- .../models/node_execution_event.go | 8 - pkg/repositories/models/project.go | 10 - pkg/repositories/models/resource.go | 8 - pkg/repositories/models/task.go | 10 - pkg/repositories/models/task_execution.go | 8 - pkg/repositories/models/workflow.go | 10 - pkg/server/initialize.go | 23 +- 16 files changed, 337 insertions(+), 335 deletions(-) diff --git a/admin.db b/admin.db index eb8f7a14b8fe9d082eb7ecd6c09fcd16c5ce278d..a408f73beabe25cba94fcb84e3c52193e8d34f38 100644 GIT binary patch delta 2379 zcmbVO3v5$W7{32Kx4o~n_YQ}56XrJBv2|r^tebOA+{zFf0|px7bdy=w3{x3wD>#G7 zI#>+gW8@_JVI21RATiO11fxL(6N2a7>X-={warcMIrpCb zod5qGe>>0UooDo(2}$cYj;mzXZS3ON)v;-cAJ%VrV0`{*=W>VFIa7UEovECZ?^hzy zG3g%hls!Xy(bi|1DD*(5PzN_~{cIJtDi!0$As;uo$!xhXMKt`G5C#k$gJj%!90c5X z1QdMXDBNfc)0FPo>qR3o6E>`-o8T7I2BCWQ0hr6v2BDaQgTXHyi$LiDA{uFFu=Z|z;teRWDl-^!O@+0;XwhFZ z(0eLpaoQ_d$hE>%?ldv-sTb7;m7kP0>4LORDifok&la<-5zh1Tco*EmeZ@kcDQ@lqH(q`hbnJ*g2214m2;79XN7(GQyTFN8A}~P+7kNaSHv+RTZax_4 zgE{u;Zo9G2M*~_4?$`}VbVfh8dB>9AlBG=z?V(_3WjoeyWgGlB4D<2gR`^(6nl2i7 zdC*PplWj29%BuGO%*KK3u=ao9O}B`p|AhI#zsH(sp~cwqKj>ddm5maMZbjoO3GtNj?CR9j6_O)bs2`&h{&Qa2!&x zU=m5jb`QDHyi@S>dkl-R-s6zMFP>`=jGk3*lT+^agEg2p0ApxFzjQ2#mc0qD0q)%m zUuj+U%@B=}5?FUvD7g5MCJgL?@wj*h0#?!S`yqHke%vP-J|C>l$Ns%gWl=E7w9?Eg z;Z(0^)RaKD6#F7zpf(I5*6aZZ$L)crwpP!!f_IlEg2ONi{?*%m1h)`H+3L9D2&tFU4eB`VBKvZ# zB90TjPyCuLH|E3dgZr>=ge7)XojxYfbb;-07e2R*9q3XfNlmL{<8ek_cjF360UK6_ zg59<3{!jBuMcfaW7C6iSWuk&#&QpPNLQWLAq$uW9Sd^b@wuUKZNSwRs7vpLhbiY>xbf ztw$jjUFIG?xQU1{)(CjAk?P>#S9K&>hjjwBB9}ZW8i@rZ?~Pe7k$-FzbwL5{J;G)e zd5DUeJtRX2^D9|EGY9J&W)A$SkQsAALRwbWxhk!8Cav6r-c}rebh)wZZ@N|c3AT^f zt$2Sc4GM)qWmu`fb0?q_ZpHT_FbUadb*zmuz0SwS;&3x&#S<1O;umqqlH2BqMyU_N zPE$;&*cXQ)XvT$cNVPRLKHyabq2Nk>e@yR>74*k?-i)agsAdp1D;eB6xKLLBKg=Wn zL8=!%W2KskRz0p}&gK^-GPg=%p(oXo&qmQpW+ea*NgFj5#?E>YG~vuiRLiOWi_>~d zU`PA4c?8uA8!`jX1Ujrf0fh}cW4PRyMMk+i5lGnop9D(&D(?j*pQ6~B&9MDf3a|Kk zXY*Wjw&s?`rVi#i4K2$Xn;M!rx*saFq`6VVj69NPPS&51-U6b&q!#E=wSfG*roV#W Wq;Fav+JO1M^r$07E|mAjGJXfn>(eU$ delta 2320 zcmb7GZE#fO6~51X@9zEBZ?f?w&@@>W2w}5f^Hrn-vT?;25~_f-Vn*2&GZci3B})vp zCTt*x@WaGwbNt~zc8d~g8^{D*s7XkN5BZSbFglg~K!zXfv?J{w%S>y*zIQhcW5&wx z#FZ7jGVozT*xA@PFP4;3(t?i_3 zwRTy3T05XzRhs0_th40tHulx}TS}=gB%HJgki7U&IgTqK5^vo;CN$ zy3dEb+l;;VF%6{fYWk}JW)Y2MsaovJr6q%?G)_w{>Ka51OQ>EdJA?}4QtS|(K#oyp zsWLvb6p?cfL9M&PBkObKpr?{<4{)JM1u$TBX=! zqv$At#%;|{M@F$gY}IIZ0`p)qzSbUQDv$fo2}HH-Tv^t29lKPc(jJ&@?&>hs8xJ|Y z(h{9a)(Z--^HF+n2vu3Ve-Gi8M#MEj2M$BodDE4?>4LkrYL(!6#}#+BIUjUfcWkwn z+iu%BwJ)>|<(6_v3Cl<2D$7TfebRr#wcK=ffqL(1k zg(M)PV6__nraQM{!iPM@E|GTM8L+yCMqK2=)^Wjz?g zu9xWiTWB(aew7B_#?E_1fw5m+bax~S-y0IlGJiiB(i{y5d}r;_ymksIt$l)5s1O{- zl}`D(!>hevU1fRG;*$2MOJ%YiXMYhV#l`TeIj%Q_6jzM2Hf2y`{K@9$XjF~)wwYpA zV+Lw;Wdeot(jm;C@lp6VvL2a27KMf}9a?D)Etg2PF&tzGETHfS{0J2^K92h-dV*JO z-~@aup_S3CF)X67abCi*JXR(pDuXgb^H>En<}*KC8^=@BpU-No$~j@)%777*@~AXr zR&WGY>F^kqqmmkr&<_RQuMT6vT=_dkxJE(`k?-@}l)Js|6W0A4Af^7qkjr*G1{tv8C= zPDgRZM9(6s`2(6w`F?uf1N34wH7D_~qh#uf4bLpkwxS=38U20z!6|uL@lb+gt+slbD86!V}t#+eYYL9-SEMulyU+Tq|4c-xfhRP?P47X-59pz zlY11(-7Dsi_(xY$*u!cHB78eT5B9{K&osc^ZSh#o8otlaBCYx3w|J4}p2C=QO{-Pk z7|Eo6-6_lx*Oim`w1pE-Z{d9zL3f01oWf$%QpstQXEXNJY0SoEI)54wEG6d|G+Umt z>dn;|GrP~AR24SKdUZ8={Pe*YJaH$_8TzFD4{;EmWok%~3!K5bj-W&p1k1^49}*U- zI}0z><}eElUBEnZS157mlaxG%VhVi3Zyd^D1^i6<(zKFKMd$gA-ul6w7uh-;j-oIH`0Xof^14F=$Pql z(+pNFwYN{fFzXNxVZO4BM?Eb9J%`mx&&IiK#`(;^$~fQNmc2jae8!b&&TrYuRk-ud zQFsyym7jczPv4m2q`Kr1Z?I1$5mHvjdOc^+5|c%XsPqy%bix#JLN0|ig3#9sQWNgf zL?|`Uo0@p6-_6QITpn5JPJwDXtWHwfc!JHgq8n#Xuc~d*_35$v^c?EBp0qs^3w}Vc zb9^FZqT#odyOe@o?44c;e&(BA3bl>BolW*8FUuxd<7d7p7{BLbcgFi#{9k6d7k&zf z>^I-RPT}w6i<{7THZ00?+CET00n3gW^RL4#rfnhiuP&bPzQ<$>cyHIF?H{u*Yg1uf HgQEWd@Ab!B diff --git a/pkg/repositories/config/migration_models.go b/pkg/repositories/config/migration_models.go index 184de5f5c..54074834c 100644 --- a/pkg/repositories/config/migration_models.go +++ b/pkg/repositories/config/migration_models.go @@ -3,6 +3,9 @@ package config import ( "time" + "github.com/google/uuid" + "gorm.io/gorm" + "github.com/flyteorg/flyteadmin/pkg/repositories/models" ) @@ -31,7 +34,10 @@ type NodeExecutionKey struct { } type NodeExecution struct { - models.BaseModel + ID uint `gorm:"type:uuid"` + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt *time.Time `gorm:"index"` NodeExecutionKey // Also stored in the closure, but defined as a separate column because it's useful for filtering and sorting. Phase string @@ -84,3 +90,8 @@ type TaskExecution struct { // The child node executions (if any) launched by this task execution. ChildNodeExecution []NodeExecution `gorm:"foreignkey:ParentTaskExecutionID;references:ID"` } + +func (n *NodeExecution) BeforeCreate(tx *gorm.DB) error { + n.ID = uint(uuid.New().ID()) + return nil +} diff --git a/pkg/repositories/config/migrations.go b/pkg/repositories/config/migrations.go index 65a69473e..67833478b 100644 --- a/pkg/repositories/config/migrations.go +++ b/pkg/repositories/config/migrations.go @@ -17,365 +17,437 @@ var ( "schedule_entities_snapshots", "task_executions", "tasks", "workflows"} ) -var Migrations = []*gormigrate.Migration{ +type MigrationOptions struct { + IgnoreForSqlite bool +} + +type Migration struct { + gormigrate.Migration + Options MigrationOptions +} + +var Migrations = []*Migration{ // Create projects table. { - ID: "2019-05-22-projects", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Project{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Migrator().DropTable("projects") + Migration: gormigrate.Migration{ + ID: "2019-05-22-projects", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Project{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable("projects") + }, }, }, // Create Task { - ID: "2018-05-23-tasks", - Migrate: func(tx *gorm.DB) error { - // The gormigrate library recommends that we copy the actual struct into here for record-keeping but after - // some internal discussion we've decided that that's not necessary. Just a history of what we've touched - // when should be sufficient. - return tx.AutoMigrate(&models.Task{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Migrator().DropTable("tasks") + Migration: gormigrate.Migration{ + ID: "2018-05-23-tasks", + Migrate: func(tx *gorm.DB) error { + // The gormigrate library recommends that we copy the actual struct into here for record-keeping but after + // some internal discussion we've decided that that's not necessary. Just a history of what we've touched + // when should be sufficient. + return tx.AutoMigrate(&models.Task{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable("tasks") + }, }, }, // Create Workflow { - ID: "2018-05-23-workflows", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Workflow{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Migrator().DropTable("workflows") + Migration: gormigrate.Migration{ + ID: "2018-05-23-workflows", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Workflow{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable("workflows") + }, }, }, // Create Launch Plan table { - ID: "2019-05-23-lp", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.LaunchPlan{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Migrator().DropTable("launch_plans") + Migration: gormigrate.Migration{ + ID: "2019-05-23-lp", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.LaunchPlan{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable("launch_plans") + }, }, }, // Create executions table { - ID: "2019-05-23-executions", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Execution{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Migrator().DropTable("executions") + Migration: gormigrate.Migration{ + ID: "2019-05-23-executions", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Execution{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable("executions") + }, }, }, // Create executions events table { - ID: "2019-01-29-executions-events", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.ExecutionEvent{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Migrator().DropTable("executions_events") + Migration: gormigrate.Migration{ + ID: "2019-01-29-executions-events", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.ExecutionEvent{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable("executions_events") + }, }, }, // Create node executions table { - ID: "2019-04-17-node-executions", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&NodeExecution{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Migrator().DropTable("node_executions") + Migration: gormigrate.Migration{ + ID: "2019-04-17-node-executions", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&NodeExecution{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable("node_executions") + }, }, }, // Create node executions events table { - ID: "2019-01-29-node-executions-events", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.NodeExecutionEvent{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Migrator().DropTable("node_executions_events") + Migration: gormigrate.Migration{ + ID: "2019-01-29-node-executions-events", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.NodeExecutionEvent{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable("node_executions_events") + }, }, }, // Create task executions table { - ID: "2019-03-16-task-executions", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&TaskExecution{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Migrator().DropTable("task_executions") + Migration: gormigrate.Migration{ + ID: "2019-03-16-task-executions", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&TaskExecution{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable("task_executions") + }, }, }, // Update node executions with null parent values { - ID: "2019-04-17-node-executions-backfill", - Migrate: func(tx *gorm.DB) error { - return tx.Exec("update node_executions set parent_task_execution_id = NULL where parent_task_execution_id = 0").Error + Migration: gormigrate.Migration{ + ID: "2019-04-17-node-executions-backfill", + Migrate: func(tx *gorm.DB) error { + return tx.Exec("update node_executions set parent_task_execution_id = NULL where parent_task_execution_id = 0").Error + }, }, }, // Update executions table to add cluster { - ID: "2019-09-27-executions", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Execution{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Exec("ALTER TABLE executions DROP COLUMN IF EXISTS cluster").Error + Migration: gormigrate.Migration{ + ID: "2019-09-27-executions", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Execution{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec("ALTER TABLE executions DROP COLUMN IF EXISTS cluster").Error + }, }, }, // Update projects table to add description column { - ID: "2019-10-09-project-description", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Project{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Exec("ALTER TABLE projects DROP COLUMN IF EXISTS description").Error + Migration: gormigrate.Migration{ + ID: "2019-10-09-project-description", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Project{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec("ALTER TABLE projects DROP COLUMN IF EXISTS description").Error + }, }, }, // Add offloaded URIs to table { - ID: "2019-10-15-offload-inputs", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Execution{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Exec("ALTER TABLE executions DROP COLUMN IF EXISTS InputsURI, DROP COLUMN IF EXISTS UserInputsURI").Error + Migration: gormigrate.Migration{ + ID: "2019-10-15-offload-inputs", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Execution{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec("ALTER TABLE executions DROP COLUMN IF EXISTS InputsURI, DROP COLUMN IF EXISTS UserInputsURI").Error + }, }, }, // Create named_entity_metadata table. { - ID: "2019-11-05-named-entity-metadata", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.NamedEntityMetadata{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Migrator().DropTable("named_entity_metadata") + Migration: gormigrate.Migration{ + ID: "2019-11-05-named-entity-metadata", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.NamedEntityMetadata{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable("named_entity_metadata") + }, }, }, // Add ProjectAttributes with custom resource attributes. { - ID: "2020-01-10-resource", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Resource{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Migrator().DropTable("resources") + Migration: gormigrate.Migration{ + ID: "2020-01-10-resource", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Resource{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable("resources") + }, }, }, // Add Type to Task model. { - ID: "2020-03-17-task-type", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Task{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Exec("ALTER TABLE tasks DROP COLUMN IF EXISTS type").Error + Migration: gormigrate.Migration{ + ID: "2020-03-17-task-type", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Task{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec("ALTER TABLE tasks DROP COLUMN IF EXISTS type").Error + }, }, }, // Add state to name entity model { - ID: "2020-04-03-named-entity-state", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.NamedEntityMetadata{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Table("named_entity_metadata").Migrator().DropColumn(&models.NamedEntityMetadata{}, "state") + Migration: gormigrate.Migration{ + ID: "2020-04-03-named-entity-state", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.NamedEntityMetadata{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Table("named_entity_metadata").Migrator().DropColumn(&models.NamedEntityMetadata{}, "state") + }, }, }, // Set default state value for workflow model { - ID: "2020-04-03-named-entity-state-default", - Migrate: func(tx *gorm.DB) error { - return tx.Exec("UPDATE named_entity_metadata SET state = 0").Error + Migration: gormigrate.Migration{ + ID: "2020-04-03-named-entity-state-default", + Migrate: func(tx *gorm.DB) error { + return tx.Exec("UPDATE named_entity_metadata SET state = 0").Error + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec("UPDATE named_entity_metadata set state = NULL").Error + }, }, - Rollback: func(tx *gorm.DB) error { - return tx.Exec("UPDATE named_entity_metadata set state = NULL").Error + }, + // Modify the workflows table, if necessary + { + Migration: gormigrate.Migration{ + ID: "2020-04-03-workflow-state", + Migrate: func(tx *gorm.DB) error { + return tx.Exec("ALTER TABLE workflows DROP COLUMN IF EXISTS state").Error + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec("ALTER TABLE workflows ADD COLUMN IF NOT EXISTS state integer;").Error + }, }, + Options: MigrationOptions{IgnoreForSqlite: true}, }, // Modify the executions & node_execution table, if necessary { - ID: "2020-04-29-executions", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Execution{}, &models.NodeExecution{}) - }, - Rollback: func(tx *gorm.DB) error { - if err := tx.Model(&models.Execution{}).Migrator().DropColumn(&models.Execution{}, "error_code"); err != nil { - return err - } - if err := tx.Model(&models.Execution{}).Migrator().DropColumn(&models.Execution{}, "error_kind"); err != nil { - return err - } - if err := tx.Model(&models.NodeExecution{}).Migrator().DropColumn(&models.NodeExecution{}, "error_code"); err != nil { - return err - } - if err := tx.Model(&models.NodeExecution{}).Migrator().DropColumn(&models.NodeExecution{}, "error_kind"); err != nil { - return err - } - return nil + Migration: gormigrate.Migration{ + ID: "2020-04-29-executions", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Execution{}, &models.NodeExecution{}) + }, + Rollback: func(tx *gorm.DB) error { + if err := tx.Model(&models.Execution{}).Migrator().DropColumn(&models.Execution{}, "error_code"); err != nil { + return err + } + if err := tx.Model(&models.Execution{}).Migrator().DropColumn(&models.Execution{}, "error_kind"); err != nil { + return err + } + if err := tx.Model(&models.NodeExecution{}).Migrator().DropColumn(&models.NodeExecution{}, "error_code"); err != nil { + return err + } + if err := tx.Model(&models.NodeExecution{}).Migrator().DropColumn(&models.NodeExecution{}, "error_kind"); err != nil { + return err + } + return nil + }, }, }, // Add TaskID to Execution model. { - ID: "2020-04-14-task-type", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Execution{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Exec("ALTER TABLE executions DROP COLUMN IF EXISTS task_id").Error + Migration: gormigrate.Migration{ + ID: "2020-04-14-task-type", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Execution{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec("ALTER TABLE executions DROP COLUMN IF EXISTS task_id").Error + }, }, }, // NodeExecutions table has CacheStatus for Task nodes { - ID: "2020-07-27-cachestatus", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.NodeExecution{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Model(&models.NodeExecution{}).Migrator().DropColumn(&models.NodeExecution{}, "cache_status") + Migration: gormigrate.Migration{ + ID: "2020-07-27-cachestatus", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.NodeExecution{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Model(&models.NodeExecution{}).Migrator().DropColumn(&models.NodeExecution{}, "cache_status") + }, }, }, { - ID: "2020-07-31-node-execution", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.NodeExecution{}) - }, - Rollback: func(tx *gorm.DB) error { - if err := tx.Model(&models.NodeExecution{}).Migrator().DropColumn(&models.NodeExecution{}, "parent_id"); err != nil { - return err - } - return tx.Model(&models.NodeExecution{}).Migrator().DropColumn(&models.NodeExecution{}, "node_execution_metadata") + Migration: gormigrate.Migration{ + ID: "2020-07-31-node-execution", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.NodeExecution{}) + }, + Rollback: func(tx *gorm.DB) error { + if err := tx.Model(&models.NodeExecution{}).Migrator().DropColumn(&models.NodeExecution{}, "parent_id"); err != nil { + return err + } + return tx.Model(&models.NodeExecution{}).Migrator().DropColumn(&models.NodeExecution{}, "node_execution_metadata") + }, }, }, { - ID: "2020-08-17-labels-addition", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Project{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Model(&models.Project{}).Migrator().DropColumn(&models.Project{}, "labels") + Migration: gormigrate.Migration{ + ID: "2020-08-17-labels-addition", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Project{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Model(&models.Project{}).Migrator().DropColumn(&models.Project{}, "labels") + }, }, }, { - ID: "2020-09-01-task-exec-idx", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&TaskExecution{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Model(&TaskExecution{}).Migrator().DropIndex(&TaskExecution{}, "idx_task_executions_exec") + Migration: gormigrate.Migration{ + ID: "2020-09-01-task-exec-idx", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&TaskExecution{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Model(&TaskExecution{}).Migrator().DropIndex(&TaskExecution{}, "idx_task_executions_exec") + }, }, }, { - ID: "2020-11-03-project-state-addition", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Project{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Model(&models.Project{}).Migrator().DropColumn(&models.Project{}, "state") + Migration: gormigrate.Migration{ + ID: "2020-11-03-project-state-addition", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Project{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Model(&models.Project{}).Migrator().DropColumn(&models.Project{}, "state") + }, }, }, { - ID: "2020-11-03-project-state-default", - Migrate: func(tx *gorm.DB) error { - return tx.Exec("UPDATE projects set state = 0").Error - }, - Rollback: func(tx *gorm.DB) error { - return tx.Exec("UPDATE projects set state = NULL").Error + Migration: gormigrate.Migration{ + ID: "2020-11-03-project-state-default", + Migrate: func(tx *gorm.DB) error { + return tx.Exec("UPDATE projects set state = 0").Error + }, + Rollback: func(tx *gorm.DB) error { + return tx.Exec("UPDATE projects set state = NULL").Error + }, }, }, { - ID: "2021-01-22-execution-user", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Execution{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Model(&models.Execution{}).Migrator().DropColumn(&models.Execution{}, "user") + Migration: gormigrate.Migration{ + ID: "2021-01-22-execution-user", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Execution{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Model(&models.Execution{}).Migrator().DropColumn(&models.Execution{}, "user") + }, }, }, { - ID: "2021-04-19-node-execution_dynamic-workflow", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.NodeExecution{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Model(&models.NodeExecution{}).Migrator().DropColumn(&models.NodeExecution{}, "dynamic_workflow_remote_closure_reference") + Migration: gormigrate.Migration{ + ID: "2021-04-19-node-execution_dynamic-workflow", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.NodeExecution{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Model(&models.NodeExecution{}).Migrator().DropColumn(&models.NodeExecution{}, "dynamic_workflow_remote_closure_reference") + }, }, }, { - ID: "2021-07-22-schedulable_entities", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&schedulerModels.SchedulableEntity{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Migrator().DropTable(&schedulerModels.SchedulableEntity{}, "schedulable_entities") + Migration: gormigrate.Migration{ + ID: "2021-07-22-schedulable_entities", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&schedulerModels.SchedulableEntity{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable(&schedulerModels.SchedulableEntity{}, "schedulable_entities") + }, }, }, { - ID: "2021-08-05-schedulable_entities_snapshot", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&schedulerModels.ScheduleEntitiesSnapshot{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Migrator().DropTable(&schedulerModels.ScheduleEntitiesSnapshot{}, "schedulable_entities_snapshot") + Migration: gormigrate.Migration{ + ID: "2021-08-05-schedulable_entities_snapshot", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&schedulerModels.ScheduleEntitiesSnapshot{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable(&schedulerModels.ScheduleEntitiesSnapshot{}, "schedulable_entities_snapshot") + }, }, }, - // Add state to execution model. + // For any new table, Please use the following pattern due to a bug + // in the postgres gorm layer https://github.com/go-gorm/postgres/issues/65 { - ID: "2022-01-11-execution-state", - Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate(&models.Execution{}) - }, - Rollback: func(tx *gorm.DB) error { - return tx.Table("execution").Migrator().DropColumn(&models.Execution{}, "state") - }, + Migration: gormigrate.Migration{ + ID: "2022-01-11-id-to-bigint", + Migrate: func(tx *gorm.DB) error { + db, err := tx.DB() + if err != nil { + return err + } + return alterTableColumnType(db, "id", "bigint") + }, + Rollback: func(tx *gorm.DB) error { + db, err := tx.DB() + if err != nil { + return err + } + return alterTableColumnType(db, "id", "int") + }, + }, + Options: MigrationOptions{IgnoreForSqlite: true}, }, -} -// PostgresMigrations adds onto the defacto migrations. These migrations are exclusive to postgres, because: -// Databases like Sqlite don't support dropping columns. -// Databases like Sqlite don't use rigid column types, so modifying the id column type is unsupported -var PostgresMigrations = append(Migrations, []*gormigrate.Migration{ - // Modify the workflows table, if necessary - { - ID: "2020-04-03-workflow-state", - Migrate: func(tx *gorm.DB) error { - return tx.Exec("ALTER TABLE workflows DROP COLUMN IF EXISTS state").Error - }, - Rollback: func(tx *gorm.DB) error { - return tx.Exec("ALTER TABLE workflows ADD COLUMN IF NOT EXISTS state integer;").Error - }, - }, - // For any new table, Please use the following pattern due to a bug - // in the postgres gorm layer https://github.com/go-gorm/postgres/issues/65 + // Add state to execution model. { - ID: "2022-01-11-id-to-bigint", - Migrate: func(tx *gorm.DB) error { - db, err := tx.DB() - if err != nil { - return err - } - return alterTableColumnType(db, "id", "bigint") - }, - Rollback: func(tx *gorm.DB) error { - db, err := tx.DB() - if err != nil { - return err - } - return alterTableColumnType(db, "id", "int") + Migration: gormigrate.Migration{ + ID: "2022-01-11-execution-state", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&models.Execution{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Table("execution").Migrator().DropColumn(&models.Execution{}, "state") + }, }, }, -}...) +} func alterTableColumnType(db *sql.DB, columnName, columnType string) error { diff --git a/pkg/repositories/models/base_model.go b/pkg/repositories/models/base_model.go index 8660772b4..0cae391d4 100644 --- a/pkg/repositories/models/base_model.go +++ b/pkg/repositories/models/base_model.go @@ -6,7 +6,7 @@ import "time" // This is nearly identical to http://doc.gorm.io/models.html#conventions except that flyteadmin models define their // own primary keys rather than use the ID as the primary key type BaseModel struct { - ID uint `gorm:"type:uuid"` + ID uint CreatedAt time.Time UpdatedAt time.Time DeletedAt *time.Time `gorm:"index"` diff --git a/pkg/repositories/models/execution.go b/pkg/repositories/models/execution.go index 5758531ac..fa429226b 100644 --- a/pkg/repositories/models/execution.go +++ b/pkg/repositories/models/execution.go @@ -3,9 +3,6 @@ package models import ( "time" - "github.com/google/uuid" - "gorm.io/gorm" - "github.com/flyteorg/flytestdlib/storage" ) @@ -62,8 +59,3 @@ type Execution struct { // GORM doesn't save the zero value for ints, so we use a pointer for the State field State *int32 `gorm:"index;default:0"` } - -func (e *Execution) BeforeCreate(tx *gorm.DB) error { - e.ID = uint(uuid.New().ID()) - return nil -} diff --git a/pkg/repositories/models/execution_event.go b/pkg/repositories/models/execution_event.go index ef4fd3990..b6e18decc 100644 --- a/pkg/repositories/models/execution_event.go +++ b/pkg/repositories/models/execution_event.go @@ -2,9 +2,6 @@ package models import ( "time" - - "github.com/google/uuid" - "gorm.io/gorm" ) type ExecutionEvent struct { @@ -14,8 +11,3 @@ type ExecutionEvent struct { OccurredAt time.Time Phase string `gorm:"primary_key"` } - -func (e *ExecutionEvent) BeforeCreate(tx *gorm.DB) error { - e.ID = uint(uuid.New().ID()) - return nil -} diff --git a/pkg/repositories/models/launch_plan.go b/pkg/repositories/models/launch_plan.go index 5a411287a..2b2787518 100644 --- a/pkg/repositories/models/launch_plan.go +++ b/pkg/repositories/models/launch_plan.go @@ -1,10 +1,5 @@ package models -import ( - "github.com/google/uuid" - "gorm.io/gorm" -) - // Launch plan primary key type LaunchPlanKey struct { Project string `gorm:"primary_key;index:lp_project_domain_name_idx,lp_project_domain_idx" valid:"length(0|255)"` @@ -37,8 +32,3 @@ type LaunchPlan struct { Digest []byte ScheduleType LaunchPlanScheduleType } - -func (l *LaunchPlan) BeforeCreate(tx *gorm.DB) error { - l.ID = uint(uuid.New().ID()) - return nil -} diff --git a/pkg/repositories/models/named_entity.go b/pkg/repositories/models/named_entity.go index c73c37473..966676d0c 100644 --- a/pkg/repositories/models/named_entity.go +++ b/pkg/repositories/models/named_entity.go @@ -2,8 +2,6 @@ package models import ( "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" - "github.com/google/uuid" - "gorm.io/gorm" ) // NamedEntityMetadata primary key @@ -42,8 +40,3 @@ type NamedEntity struct { NamedEntityKey NamedEntityMetadataFields } - -func (n *NamedEntityMetadata) BeforeCreate(tx *gorm.DB) error { - n.ID = uint(uuid.New().ID()) - return nil -} diff --git a/pkg/repositories/models/node_execution.go b/pkg/repositories/models/node_execution.go index f1f9dab08..c9fb4a368 100644 --- a/pkg/repositories/models/node_execution.go +++ b/pkg/repositories/models/node_execution.go @@ -17,7 +17,10 @@ type NodeExecutionKey struct { // By convention, gorm foreign key references are of the form {ModelName}ID type NodeExecution struct { - BaseModel + ID uint `gorm:"type:uuid;index"` + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt *time.Time `gorm:"index"` NodeExecutionKey // Also stored in the closure, but defined as a separate column because it's useful for filtering and sorting. Phase string diff --git a/pkg/repositories/models/node_execution_event.go b/pkg/repositories/models/node_execution_event.go index 2ee001b4a..cd362c8ff 100644 --- a/pkg/repositories/models/node_execution_event.go +++ b/pkg/repositories/models/node_execution_event.go @@ -2,9 +2,6 @@ package models import ( "time" - - "github.com/google/uuid" - "gorm.io/gorm" ) type NodeExecutionEvent struct { @@ -14,8 +11,3 @@ type NodeExecutionEvent struct { OccurredAt time.Time Phase string `gorm:"primary_key"` } - -func (n *NodeExecutionEvent) BeforeCreate(tx *gorm.DB) error { - n.ID = uint(uuid.New().ID()) - return nil -} diff --git a/pkg/repositories/models/project.go b/pkg/repositories/models/project.go index a35985eb3..a5feedb27 100644 --- a/pkg/repositories/models/project.go +++ b/pkg/repositories/models/project.go @@ -1,10 +1,5 @@ package models -import ( - "github.com/google/uuid" - "gorm.io/gorm" -) - type Project struct { BaseModel Identifier string `gorm:"primary_key"` @@ -14,8 +9,3 @@ type Project struct { // GORM doesn't save the zero value for ints, so we use a pointer for the State field State *int32 `gorm:"default:0;index"` } - -func (p *Project) BeforeCreate(tx *gorm.DB) error { - p.ID = uint(uuid.New().ID()) - return nil -} diff --git a/pkg/repositories/models/resource.go b/pkg/repositories/models/resource.go index 8d274cf37..4512038c3 100644 --- a/pkg/repositories/models/resource.go +++ b/pkg/repositories/models/resource.go @@ -2,9 +2,6 @@ package models import ( "time" - - "github.com/google/uuid" - "gorm.io/gorm" ) type ResourcePriority int32 @@ -32,8 +29,3 @@ type Resource struct { // Serialized flyteidl.admin.MatchingAttributes. Attributes []byte } - -func (r *Resource) BeforeCreate(tx *gorm.DB) error { - r.ID = int64(uuid.New().ID()) - return nil -} diff --git a/pkg/repositories/models/task.go b/pkg/repositories/models/task.go index 09de5e12d..093bd29ec 100644 --- a/pkg/repositories/models/task.go +++ b/pkg/repositories/models/task.go @@ -1,10 +1,5 @@ package models -import ( - "github.com/google/uuid" - "gorm.io/gorm" -) - // IMPORTANT: If you update the model below, be sure to double check model definitions in // pkg/repositories/config/migration_models.go @@ -26,8 +21,3 @@ type Task struct { // Task type (also stored in the closure put promoted as a column for filtering). Type string `valid:"length(0|255)"` } - -func (t *Task) BeforeCreate(tx *gorm.DB) error { - t.ID = uint(uuid.New().ID()) - return nil -} diff --git a/pkg/repositories/models/task_execution.go b/pkg/repositories/models/task_execution.go index 271504812..d1f8ad243 100644 --- a/pkg/repositories/models/task_execution.go +++ b/pkg/repositories/models/task_execution.go @@ -2,9 +2,6 @@ package models import ( "time" - - "github.com/google/uuid" - "gorm.io/gorm" ) // IMPORTANT: If you update the model below, be sure to double check model definitions in @@ -40,8 +37,3 @@ type TaskExecution struct { // The child node executions (if any) launched by this task execution. ChildNodeExecution []NodeExecution `gorm:"foreignkey:ParentTaskExecutionID;references:ID"` } - -func (t *TaskExecution) BeforeCreate(tx *gorm.DB) error { - t.ID = uint(uuid.New().ID()) - return nil -} diff --git a/pkg/repositories/models/workflow.go b/pkg/repositories/models/workflow.go index 5d299e4ca..5f50379b1 100644 --- a/pkg/repositories/models/workflow.go +++ b/pkg/repositories/models/workflow.go @@ -1,10 +1,5 @@ package models -import ( - "github.com/google/uuid" - "gorm.io/gorm" -) - // Workflow primary key type WorkflowKey struct { Project string `gorm:"primary_key;index:workflow_project_domain_name_idx;index:workflow_project_domain_idx" valid:"length(0|255)"` @@ -22,8 +17,3 @@ type Workflow struct { // Hash of the compiled workflow closure Digest []byte } - -func (w *Workflow) BeforeCreate(tx *gorm.DB) error { - w.ID = uint(uuid.New().ID()) - return nil -} diff --git a/pkg/server/initialize.go b/pkg/server/initialize.go index 7b4c61fea..baa872f09 100644 --- a/pkg/server/initialize.go +++ b/pkg/server/initialize.go @@ -40,14 +40,21 @@ func withDB(ctx context.Context, do func(db *gorm.DB, dbType repositories.Databa return do(db, dbType) } +func getMigrations(dbType repositories.DatabaseType) []*gormigrate.Migration { + var migrations = make([]*gormigrate.Migration, 0, len(config.Migrations)) + for _, migration := range config.Migrations { + if dbType == repositories.DatabaseTypeSqlite && migration.Options.IgnoreForSqlite { + continue + } + migrations = append(migrations, &migration.Migration) + } + return migrations +} + // Migrate runs all configured migrations func Migrate(ctx context.Context) error { return withDB(ctx, func(db *gorm.DB, dbType repositories.DatabaseType) error { - migrations := config.Migrations - if dbType == repositories.DatabaseTypePostgres { - migrations = config.PostgresMigrations - } - m := gormigrate.New(db.Debug(), gormigrate.DefaultOptions, migrations) + m := gormigrate.New(db.Debug(), gormigrate.DefaultOptions, getMigrations(dbType)) if err := m.Migrate(); err != nil { return fmt.Errorf("database migration failed: %v", err) } @@ -59,11 +66,7 @@ func Migrate(ctx context.Context) error { // Rollback rolls back the last migration func Rollback(ctx context.Context) error { return withDB(ctx, func(db *gorm.DB, dbType repositories.DatabaseType) error { - migrations := config.Migrations - if dbType == repositories.DatabaseTypePostgres { - migrations = config.PostgresMigrations - } - m := gormigrate.New(db, gormigrate.DefaultOptions, migrations) + m := gormigrate.New(db, gormigrate.DefaultOptions, getMigrations(dbType)) err := m.RollbackLast() if err != nil { return fmt.Errorf("could not rollback latest migration: %v", err) From 2cb94d5c7c5c8422943266498f5b2e5e9e1ba213 Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Tue, 22 Mar 2022 16:34:14 -0700 Subject: [PATCH 13/14] tests Signed-off-by: Katrina Rogan --- pkg/manager/impl/execution_manager_test.go | 8 ++------ pkg/manager/impl/node_execution_manager_test.go | 4 +--- pkg/manager/impl/task_execution_manager_test.go | 4 +--- pkg/repositories/gormimpl/node_execution_repo_test.go | 6 ++---- pkg/repositories/gormimpl/resource_repo_test.go | 4 ++-- 5 files changed, 8 insertions(+), 18 deletions(-) diff --git a/pkg/manager/impl/execution_manager_test.go b/pkg/manager/impl/execution_manager_test.go index c0c016bc1..faae6e398 100644 --- a/pkg/manager/impl/execution_manager_test.go +++ b/pkg/manager/impl/execution_manager_test.go @@ -372,9 +372,7 @@ func TestCreateExecutionFromWorkflowNode(t *testing.T) { assert.EqualValues(t, input.NodeExecutionIdentifier, parentNodeExecutionID) getNodeExecutionCalled = true return models.NodeExecution{ - BaseModel: models.BaseModel{ - ID: 1, - }, + ID: 1, }, nil }, ) @@ -1197,9 +1195,7 @@ func TestRecoverExecution_RecoveredChildNode(t *testing.T) { assert.True(t, proto.Equal(&parentNodeExecution, &input.NodeExecutionIdentifier)) return models.NodeExecution{ - BaseModel: models.BaseModel{ - ID: parentNodeDatabaseID, - }, + ID: parentNodeDatabaseID, }, nil }) diff --git a/pkg/manager/impl/node_execution_manager_test.go b/pkg/manager/impl/node_execution_manager_test.go index d88899d6a..20ce47a6b 100644 --- a/pkg/manager/impl/node_execution_manager_test.go +++ b/pkg/manager/impl/node_execution_manager_test.go @@ -657,9 +657,7 @@ func TestListNodeExecutionsWithParent(t *testing.T) { repository.NodeExecutionRepo().(*repositoryMocks.MockNodeExecutionRepo).SetGetCallback(func(ctx context.Context, input interfaces.NodeExecutionResource) (execution models.NodeExecution, e error) { assert.Equal(t, "parent_1", input.NodeExecutionIdentifier.NodeId) return models.NodeExecution{ - BaseModel: models.BaseModel{ - ID: parentID, - }, + ID: parentID, }, nil }) repository.NodeExecutionRepo().(*repositoryMocks.MockNodeExecutionRepo).SetListCallback( diff --git a/pkg/manager/impl/task_execution_manager_test.go b/pkg/manager/impl/task_execution_manager_test.go index 13d11c9d6..a30bb34b9 100644 --- a/pkg/manager/impl/task_execution_manager_test.go +++ b/pkg/manager/impl/task_execution_manager_test.go @@ -540,9 +540,7 @@ func TestGetTaskExecution(t *testing.T) { Closure: closureBytes, ChildNodeExecution: []models.NodeExecution{ { - BaseModel: models.BaseModel{ - ID: uint(2), - }, + ID: uint(2), }, }, }, nil diff --git a/pkg/repositories/gormimpl/node_execution_repo_test.go b/pkg/repositories/gormimpl/node_execution_repo_test.go index a2679a919..fc2f1cfa1 100644 --- a/pkg/repositories/gormimpl/node_execution_repo_test.go +++ b/pkg/repositories/gormimpl/node_execution_repo_test.go @@ -65,7 +65,7 @@ func TestUpdateNodeExecution(t *testing.T) { nodeExecutionQuery.WithQuery(`UPDATE "node_executions" SET "id"=$1,"updated_at"=$2,"execution_project"=$3,"execution_domain"=$4,"execution_name"=$5,"node_id"=$6,"phase"=$7,"input_uri"=$8,"closure"=$9,"started_at"=$10,"node_execution_created_at"=$11,"node_execution_updated_at"=$12,"duration"=$13 WHERE "execution_project" = $14 AND "execution_domain" = $15 AND "execution_name" = $16 AND "node_id" = $17`) err := nodeExecutionRepo.Update(context.Background(), &models.NodeExecution{ - BaseModel: models.BaseModel{ID: 1}, + ID: 1, NodeExecutionKey: models.NodeExecutionKey{ NodeID: "1", ExecutionKey: models.ExecutionKey{ @@ -381,9 +381,7 @@ func TestNodeExecutionExists(t *testing.T) { Name: "1", }, }, - BaseModel: models.BaseModel{ - ID: id, - }, + ID: id, Phase: nodePhase, Closure: []byte("closure"), } diff --git a/pkg/repositories/gormimpl/resource_repo_test.go b/pkg/repositories/gormimpl/resource_repo_test.go index 5c2753580..00ecb7d41 100644 --- a/pkg/repositories/gormimpl/resource_repo_test.go +++ b/pkg/repositories/gormimpl/resource_repo_test.go @@ -22,7 +22,7 @@ func TestCreateWorkflowAttributes(t *testing.T) { query := GlobalMock.NewMock() GlobalMock.Logging = true query.WithQuery( - `INSERT INTO "resources" ("created_at","updated_at","deleted_at","project","domain","workflow","launch_plan","resource_type","priority","attributes","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING "id"`) + `INSERT INTO "resources" ("created_at","updated_at","deleted_at","project","domain","workflow","launch_plan","resource_type","priority","attributes") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING "id"`) err := resourceRepo.CreateOrUpdate(context.Background(), models.Resource{ Project: "project", @@ -65,7 +65,7 @@ func TestUpdateWorkflowAttributes_WithExisting(t *testing.T) { mockSaveQuery := GlobalMock.NewMock() mockSaveQuery.WithQuery( - `INSERT INTO "resources" ("created_at","updated_at","deleted_at","project","domain","workflow","launch_plan","resource_type","priority","attributes","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)`) + `INSERT INTO "resources" ("created_at","updated_at","deleted_at","project","domain","workflow","launch_plan","resource_type","priority","attributes") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`) err := resourceRepo.CreateOrUpdate(context.Background(), models.Resource{ ResourceType: resourceType.String(), From 72ef3ff3c9394ddda734046bbe223f789bc71443 Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Wed, 23 Mar 2022 12:52:46 -0700 Subject: [PATCH 14/14] something Signed-off-by: Katrina Rogan --- admin.db | Bin 274432 -> 319488 bytes pkg/repositories/config/migration_models.go | 76 +++++++++++------- pkg/repositories/config/migrations.go | 24 +++--- .../gormimpl/node_execution_repo_test.go | 2 +- pkg/repositories/models/base_model.go | 2 +- pkg/repositories/models/execution.go | 6 +- pkg/repositories/models/execution_event.go | 2 +- pkg/repositories/models/launch_plan.go | 8 +- pkg/repositories/models/named_entity.go | 8 +- pkg/repositories/models/node_execution.go | 13 +-- .../models/node_execution_event.go | 2 +- pkg/repositories/models/project.go | 2 +- pkg/repositories/models/task.go | 8 +- pkg/repositories/models/workflow.go | 8 +- 14 files changed, 85 insertions(+), 76 deletions(-) diff --git a/admin.db b/admin.db index a408f73beabe25cba94fcb84e3c52193e8d34f38..b858ef3a8456e6a613b93177c05c65fc77081707 100644 GIT binary patch delta 5580 zcmeHLdr(y872mV>?!9~OzIa~{7k0rFmUVfE2u6(%LMc%RsKiH-va76-ihwMls1+}o z=d@|eoq9TxX%vhDnKZs5l0+q~S`w``k7=8B(%Sryw)rEE&NPonqCNNC-9>DmcG^EW zZD;4(x%d0d`M&SGe&?LK^J?7Ak;Es`;|B}|LoE#t4WO}i&y3=V0YmtlYXUFT~ zhCo=G#!JN>kO5yAfD~(UQ%i7D>zZ|q?JdDQuO};iGTDsHP0DpsGS}P`@a{;(`GZgl zX}EC^?y8qLiI@DuUESQ;6x!0>(Yztp(cRe6-l)BgZPgl1tz1%7v8>9~P+L>qQ01zr zy|ZeCD{wQ1dJsN60IBF31q&k!DVRS1GeN@O0IYyyCDSxZInO$<9TULCdiA&=^@IZYa{v=PMA~`Q8JB5 zDH41URXF05%jObAFz4Wrm%*;gF+1_2=fG^-;ZWY@3zdh=4t)MONL32WIa$3~c3#TN zgx!gq!PPCH=$_%&gG1>Nt_hlsR@-<9?;e3{u;ZCCkh1qEOkcF!%1cEO$n(bu)ENw~ z*M(7^!nR2Ti4iMc&C!G~_E<3RGWeBiVzw$ofw5J@qa%qpK#pOn zwZZZZF=#)fK5kx%oado9_d_o~&K*J_UH;IP(3(x1&26oXp)QgpoI8OYbaaSv>8iMy zH=wxLvcwsI_I1HUCE`P=$4-G2Prn2_ zBc~N%?J(Q{CF;rdh`kF7d8xD%c9-D17oh-V4Yj-oj;OFuz)O|nJ8t#fX;{!lX83A9 ztB)r%Mm7*-p z;H4U3^Xq&6@G_)9fxhhN|0NoIE>X;Z~zWt>obrc2HJxiRAi`C;hP6YzKxm8w1hRmHS0o+;m%;^ zrf|U38QRi`*AKwVq`DC!RLD;|f!>ur1LEbeI(=Gio|Xt-&(Xk^mJ zppj0)emuVLEl2+d6;kC|Emc@%BSeX3rx=7&!YuO+c7UD2jdIzhy^buqY-_gu(()V2 z_bop0d2xoh(QJT4hW8BXS()voDyo=eKWFVI$20kmt-`nye;&{H*rv@?e)qi!IRquz z>_8&pWV^a(R!(62K;EO$_8b9k8vD8N1Jre}W>oLiM{uTzeX{{*uajjC%o6g-HI&szcjpsDkns zS&gH&Kw3g`Qw*l{Ah=D(+^5gN6s=?KSv9`pS+srSEp%3v zNkd6HQq^Xxvtn1T(3`kGX3g>mnkbbkENCrWq1yUxCXrQ z7`UMd*B*m*Scbnk1{rWCesT;x&B$0v)l0);A%zPP85175NZFj8GO1*O zIt}8`cL_33131#-`&gdDqB05P}`-sJS()iLi1JuS0AQ;65WbsA_(C%v) zQ;%klc@{ZKHtLYC_ASn!Cfj-j(XgJ3*h9{u#+>k z<7?-k5^B`7RpKu5R+69u9xTR}E`T4V5B>Q9h%0nwxy^WLZEm4AODn3N3 zwrW!EIvR$q?s5JUaln{AaiI;eKopvJB7Ufv$J8y;*B0GAkz}RX$J8YAG_B>ZNVZZ< zZ0eG^8;`-ti%@Nnjj~DidpvcKoI&;aRq_;>y;a}M_~(6>^LEMfAw|unox=UH@diJp zgfM3Jh|j5s!mBF%L5~dl*XG+Xp4y?dSD?qJ8Ohsvul_m9rhd)>tn B^ymNp delta 4022 zcmbtXe^3z_+~zpoL_`zc@%-GQJs za46%bnTcjQ^PTA*(K6Fa+jPbd|6n?4tLT_0qNdZdiFKlMI&C@=N7FWAVlvb8?HwT6 zkZ6Q?Z{L3Ry?5W;_xXI@=MA1-FgRsCvN+`+!!Wh9JV^^nOW)Cg67NB#uFb!y4T$bZdn7wOfUsX)bCu@*k*e{A;S?swLzI43f=|!Hg(Fdebpw zFIkMA2FXImz?2ZlRN4X->?BsU&4PyAL;~7Lc~E5MGKJyi7xALq4u|&OWjmJ+A~tpq z3x0AIn(_8|s2e-hP1*s2og{$&adOL7N>Yia7VGT5bcEUhebMe;9w2&1lV2u{>`M#e zakc~&(i^LMofYIP&h8xAlfsJ*2Mm|T{M&7+V1NL`a{vV?bNqaOF@qCITcb_ zs#rU zoyO;xZ5l&5*`ssH9|<`$A1(f4fMl@S1^Kep0Xq5jI*Y6qa@hf`{HEXrLB1|DWeiz( zUKB;xZ%TZum_OPXO;j40=s!{tDUc+MTC5c3OC-_T=Sf_&a;2e*-KLULjSf)DI>TbQ z*_bH|0mqA(nXvC!dAIR)MIaRG*wGP)v`A7BQH!;i^G)3q3BMR5m(9v?6fOMvCCMM;D}52 zg>HxNZy}=jO!Ky86+gziRHsz~MGNWV6e1@wEx4gZ$)!@C1Eex(K!Pl{sp$$u-}5hQe)u``GY6 zAa6J;n-^U!I2gsBUxFO`*v=X8&zHbLdU`tAaMaDMQ7HHvzH%9k#36viz zSihluUCo9~wg&Ge8~)45)#{|U(awtvZisu(bC!z9uipV7WtP#tKrfvjnyX03Vm$F% zC~K&)sl{gZe4pdNk#p2xdaj(Ib}KKs-7w6dXA~TC5jKv(SBmVcK-(zrM0!jW@qcek zcvSh}q{mm0%I~2S^9he!Vd(xp3a@%(qTbJW;U4Gl%qdt-X}ocnL3fT>$edw>rP`0R z4Vu^0`KEDGrSX8_OEO_N!QC_n?E7pDk!W*(PjD6QS#t$D_4d!}Y}qZe-! zau&?X=l+Cu3pq(A?ceK461NC{bry1^`?^KIh%f6XAcu~Gd0K5f3!1UP^PmBRGS$X{macYxG(ZKJN~nYzC&9sn8kD^Rv~-)uzPFnUJeAly zG|J{tD%C&i#;_`$?d8R1%OLK>;||V>jd`3E$}pJ6ZON*yo&SyceCR3S7AfyUKe=jY zTqxqPBx|V;R zj;dfCJpd(@-qcGbJ&ZzK;IjMav}7`pX$^)cOD z-635n^9B89jLR5?e3HDEyDoY7g-c%Ix1Hn2pg8$--@WiKN78P#+~uyr#r9 zM2VHg$cJd!P4Z_Pt71%*Xha#Eo;0)5o^{sDQd_`%Sx&cc1ExgDLagW~DO5oc#}ulaiV2 zly_r$W6GRG)Ji8cE2?N#Cgw+pVI~XZ-PpEB(sGOUEcc3%?nn2}wQ=ET#jV)T&`<6B zz(p{j^9*H!ZvxCp%#3#~Kq_9}O$=K8hrC$lg1CfRCuxS%uxPPz0!*Nvbua~NWF7wX zB6vWTjFnD876>Py`0JvPDf7rwal&m?zcr<+&M&2yrch_Bv}DneF)k2F%3j-A= zo6G`ut5!ag3z!$(cFh)YRav;g