From abdd1bee849781cadd68d9ae898995ef95131771 Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Mon, 1 Jun 2026 01:54:47 +0500 Subject: [PATCH 01/16] feat(runners): allow to create unregistered runners for terraform usage --- api-docs.yml | 8 +++++++ api/runners.go | 17 +++++++++++++- api/runners/runners.go | 26 +++++++++++++-------- cli/cmd/runner_register.go | 6 +++++ config.schema.yaml | 3 +++ db/Runner.go | 10 ++++++++ db/Store.go | 3 +++ db/bolt/global_runner.go | 28 ++++++++++++++++++++++- db/bolt/global_runner_test.go | 43 +++++++++++++++++++++++++++++++++++ db/sql/global_runner.go | 37 +++++++++++++++++++++++++++++- services/runners/job_pool.go | 1 + services/runners/types.go | 6 ++++- util/config.go | 4 ++++ 13 files changed, 179 insertions(+), 13 deletions(-) diff --git a/api-docs.yml b/api-docs.yml index abc9c25305..5db73e4855 100644 --- a/api-docs.yml +++ b/api-docs.yml @@ -1223,6 +1223,14 @@ definitions: type: array items: type: string + unregistered: + type: boolean + description: > + When true, the runner is created without a token and stays inactive. + The token is not stored; the runner must be registered later via + `semaphore runner register --runner-id `. Useful for tools like + Terraform that create the runner up front and pass its ID to the + runner host via cloud-init. RunnerWithToken: allOf: diff --git a/api/runners.go b/api/runners.go index c3da13e2be..5b3c16401f 100644 --- a/api/runners.go +++ b/api/runners.go @@ -41,7 +41,12 @@ func addGlobalRunner(w http.ResponseWriter, r *http.Request) { var privateKey []byte - if runner.PublicKey == nil { + if runner.Unregistered { + // An unregistered runner has no token and cannot be active. Its keypair is + // generated by the runner itself when it registers via the CLI later on. + runner.Active = false + runner.PublicKey = nil + } else if runner.PublicKey == nil { var b bytes.Buffer privateKeyFile := bufio.NewWriter(&b) @@ -118,6 +123,11 @@ func updateGlobalRunner(w http.ResponseWriter, r *http.Request) { return } + if runner.Active && !oldRunner.IsRegistered() { + helpers.WriteErrorStatus(w, "Unregistered runner cannot be activated", http.StatusBadRequest) + return + } + store := helpers.Store(r) runner.ID = oldRunner.ID @@ -188,6 +198,11 @@ func setGlobalRunnerActive(w http.ResponseWriter, r *http.Request) { return } + if body.Active && !runner.IsRegistered() { + helpers.WriteErrorStatus(w, "Unregistered runner cannot be activated", http.StatusBadRequest) + return + } + runner.Active = body.Active err := store.UpdateRunner(*runner) diff --git a/api/runners/runners.go b/api/runners/runners.go index 6bfa4912e5..a2490ebd1c 100644 --- a/api/runners/runners.go +++ b/api/runners/runners.go @@ -351,15 +351,23 @@ func RegisterRunner(w http.ResponseWriter, r *http.Request) { return } - runner, err := helpers.Store(r).CreateRunner(db.Runner{ - Webhook: register.Webhook, - Name: register.Name, - Tags: register.Tags, - MaxParallelTasks: register.MaxParallelTasks, - Active: register.Enabled, - PublicKey: register.PublicKey, - ProjectID: register.ProjectID, - }) + var runner db.Runner + var err error + + if register.RunnerID != nil { + // Register a previously created tokenless runner by its ID. + runner, err = helpers.Store(r).RegisterRunner(*register.RunnerID, register.PublicKey, register.Enabled) + } else { + runner, err = helpers.Store(r).CreateRunner(db.Runner{ + Webhook: register.Webhook, + Name: register.Name, + Tags: register.Tags, + MaxParallelTasks: register.MaxParallelTasks, + Active: register.Enabled, + PublicKey: register.PublicKey, + ProjectID: register.ProjectID, + }) + } if err != nil { diff --git a/cli/cmd/runner_register.go b/cli/cmd/runner_register.go index 493e6e607b..14ae4f48e3 100644 --- a/cli/cmd/runner_register.go +++ b/cli/cmd/runner_register.go @@ -13,6 +13,7 @@ var runnerRegisterArgs struct { stdinRegistrationToken bool registrationTokenFilePath string projectID int + runnerID int name string tags []string webhook string @@ -30,6 +31,7 @@ func init() { runnerRegisterCmd.PersistentFlags().StringVar(&runnerRegisterArgs.registrationTokenFilePath, "registration-token-file", "", "Read registration token from a file") runnerRegisterCmd.PersistentFlags().BoolVar(&runnerRegisterArgs.enabled, "enabled", true, "Enable or disable the runner on the server") runnerRegisterCmd.PersistentFlags().IntVar(&runnerRegisterArgs.projectID, "project-id", 0, "Project ID for project-level runner (global runner if not provided)") + runnerRegisterCmd.PersistentFlags().IntVar(&runnerRegisterArgs.runnerID, "runner-id", 0, "ID of an existing unregistered runner to register (created beforehand, e.g. via Terraform)") runnerCmd.AddCommand(runnerRegisterCmd) } @@ -100,6 +102,10 @@ func registerRunner(cmd *cobra.Command) { util.Config.Runner.ProjectID = &runnerRegisterArgs.projectID } + if runnerRegisterArgs.runnerID > 0 { + util.Config.Runner.RunnerID = &runnerRegisterArgs.runnerID + } + taskPool := createRunnerJobPool() err := taskPool.Register(configFile) diff --git a/config.schema.yaml b/config.schema.yaml index d7c95cc0da..7c1f7a795f 100644 --- a/config.schema.yaml +++ b/config.schema.yaml @@ -467,3 +467,6 @@ $defs: enabled: { type: boolean } project_id: type: [integer, "null"] + runner_id: + type: [integer, "null"] + description: ID of an existing unregistered runner to register instead of creating a new one. diff --git a/db/Runner.go b/db/Runner.go index c32f2a8c5b..89a00324a6 100644 --- a/db/Runner.go +++ b/db/Runner.go @@ -30,6 +30,16 @@ type Runner struct { CleaningRequested *time.Time `db:"cleaning_requested" json:"cleaning_requested"` PublicKey *string `db:"public_key" json:"-"` + + // Unregistered is a transient flag (never persisted) used at creation time to + // request a runner without a token. Such a runner gets an empty token and must + // be registered later via `semaphore runner register --runner-id `. + Unregistered bool `db:"-" json:"unregistered"` +} + +// IsRegistered reports whether the runner has been registered (has a token). +func (r Runner) IsRegistered() bool { + return r.Token != "" } // HasTag reports whether the runner is tagged with the given tag. diff --git a/db/Store.go b/db/Store.go index 566f399978..06bb994d1a 100644 --- a/db/Store.go +++ b/db/Store.go @@ -458,6 +458,9 @@ type RunnerManager interface { DeleteGlobalRunner(runnerID int) error UpdateRunner(runner Runner) error CreateRunner(runner Runner) (Runner, error) + // RegisterRunner finalizes a previously created tokenless ("unregistered") + // runner by generating its token, storing its public key and activating it. + RegisterRunner(runnerID int, publicKey *string, active bool) (Runner, error) TouchRunner(runner Runner) (err error) ClearRunnerCache(runner Runner) (err error) GetRunnerTags(projectID int) ([]RunnerTag, error) diff --git a/db/bolt/global_runner.go b/db/bolt/global_runner.go index f79a38927b..5e328b7919 100644 --- a/db/bolt/global_runner.go +++ b/db/bolt/global_runner.go @@ -2,6 +2,7 @@ package bolt import ( "encoding/base64" + "fmt" "github.com/gorilla/securecookie" "github.com/semaphoreui/semaphore/db" @@ -131,7 +132,11 @@ func (d *BoltDb) UpdateRunner(runner db.Runner) (err error) { } func (d *BoltDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) { - runner.Token = base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32)) + if runner.Unregistered { + runner.Token = "" + } else { + runner.Token = base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32)) + } res, err := d.createObject(0, db.GlobalRunnerProps, runner) @@ -141,3 +146,24 @@ func (d *BoltDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) newRunner = res.(db.Runner) return } + +func (d *BoltDb) RegisterRunner(runnerID int, publicKey *string, active bool) (runner db.Runner, err error) { + err = d.db.Update(func(tx *bbolt.Tx) error { + err = d.getObjectTx(tx, 0, db.GlobalRunnerProps, intObjectID(runnerID), &runner) + if err != nil { + return err + } + + if runner.IsRegistered() { + return fmt.Errorf("runner is already registered") + } + + runner.Token = base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32)) + runner.PublicKey = publicKey + runner.Active = active + + return d.updateObjectTx(tx, 0, db.GlobalRunnerProps, runner) + }) + + return +} diff --git a/db/bolt/global_runner_test.go b/db/bolt/global_runner_test.go index d58e210d5e..d7362cf06f 100644 --- a/db/bolt/global_runner_test.go +++ b/db/bolt/global_runner_test.go @@ -42,3 +42,46 @@ func Test_GetGlobalRunner_ReturnsErrorWhenTryingGetProjectRunner(t *testing.T) { _, err = store.GetGlobalRunner(testRunner.ID) assert.ErrorIs(t, err, db.ErrNotFound) } + +func Test_CreateRunner_UnregisteredHasEmptyToken(t *testing.T) { + store := CreateTestStore() + + testRunner, err := store.CreateRunner(db.Runner{Unregistered: true}) + assert.NoError(t, err) + assert.Empty(t, testRunner.Token) + + stored, err := store.GetGlobalRunner(testRunner.ID) + assert.NoError(t, err) + assert.Empty(t, stored.Token) + assert.False(t, stored.IsRegistered()) +} + +func Test_RegisterRunner_SetsTokenAndActivates(t *testing.T) { + store := CreateTestStore() + + testRunner, err := store.CreateRunner(db.Runner{Unregistered: true}) + assert.NoError(t, err) + + publicKey := "test-public-key" + registered, err := store.RegisterRunner(testRunner.ID, &publicKey, true) + assert.NoError(t, err) + assert.NotEmpty(t, registered.Token) + assert.True(t, registered.Active) + assert.Equal(t, publicKey, *registered.PublicKey) + + stored, err := store.GetGlobalRunner(testRunner.ID) + assert.NoError(t, err) + assert.Equal(t, registered.Token, stored.Token) + assert.True(t, stored.Active) +} + +func Test_RegisterRunner_FailsWhenAlreadyRegistered(t *testing.T) { + store := CreateTestStore() + + testRunner, err := store.CreateRunner(db.Runner{}) + assert.NoError(t, err) + assert.NotEmpty(t, testRunner.Token) + + _, err = store.RegisterRunner(testRunner.ID, nil, true) + assert.Error(t, err) +} diff --git a/db/sql/global_runner.go b/db/sql/global_runner.go index 640dbe2b06..9ab812a117 100644 --- a/db/sql/global_runner.go +++ b/db/sql/global_runner.go @@ -172,9 +172,44 @@ func (d *SqlDb) UpdateRunner(runner db.Runner) (err error) { return } -func (d *SqlDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) { +func (d *SqlDb) RegisterRunner(runnerID int, publicKey *string, active bool) (runner db.Runner, err error) { + err = d.getObject(0, db.GlobalRunnerProps, runnerID, &runner) + if err != nil { + return + } + + if runner.IsRegistered() { + err = fmt.Errorf("runner is already registered") + return + } + token := base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32)) + _, err = d.exec( + "update `runner` set `token`=?, `public_key`=?, `active`=? where id=?", + token, + publicKey, + active, + runnerID) + + if err != nil { + return + } + + runner.Token = token + runner.PublicKey = publicKey + runner.Active = active + + err = d.loadRunnerTagsSingle(&runner) + return +} + +func (d *SqlDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) { + token := "" + if !runner.Unregistered { + token = base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32)) + } + insertID, err := d.insert( "id", "insert into `runner` (project_id, token, webhook, max_parallel_tasks, `name`, `active`, `is_default`, public_key) values (?, ?, ?, ?, ?, ?, ?, ?)", diff --git a/services/runners/job_pool.go b/services/runners/job_pool.go index c6a685ab4a..a45e84ac7c 100644 --- a/services/runners/job_pool.go +++ b/services/runners/job_pool.go @@ -397,6 +397,7 @@ func (p *JobPool) tryRegisterRunner(configFilePath *string) (ok bool) { jsonBytes, err := json.Marshal(RunnerRegistration{ RegistrationToken: util.Config.Runner.RegistrationToken, + RunnerID: util.Config.Runner.RunnerID, Webhook: util.Config.Runner.Webhook, Name: util.Config.Runner.Name, Tags: util.Config.Runner.Tags, diff --git a/services/runners/types.go b/services/runners/types.go index 05353652b8..b6279f1e9b 100644 --- a/services/runners/types.go +++ b/services/runners/types.go @@ -56,7 +56,11 @@ type JobProgress struct { } type RunnerRegistration struct { - RegistrationToken string `json:"registration_token" binding:"required"` + RegistrationToken string `json:"registration_token" binding:"required"` + // RunnerID, when set, registers a previously created tokenless ("unregistered") + // runner instead of creating a new one. Used by the Terraform provider, which + // creates the runner up front and passes its ID to the runner via cloud-init. + RunnerID *int `json:"runner_id,omitempty"` Webhook string `json:"webhook,omitempty"` Name string `json:"name,omitempty"` Tags []string `json:"tags,omitempty"` diff --git a/util/config.go b/util/config.go index dc3cb71710..835f5427c7 100644 --- a/util/config.go +++ b/util/config.go @@ -150,6 +150,10 @@ type RunnerConfig struct { MaxParallelTasks int `json:"max_parallel_tasks,omitempty" default:"1" env:"SEMAPHORE_RUNNER_MAX_PARALLEL_TASKS"` ProjectID *int `json:"project_id,omitempty" env:"SEMAPHORE_RUNNER_PROJECT_ID"` + // RunnerID, when set, registers a previously created tokenless ("unregistered") + // runner with this ID instead of creating a new runner on the server. + RunnerID *int `json:"runner_id,omitempty" env:"SEMAPHORE_RUNNER_ID"` + Connection *RunnerConnectionConfig `json:"connection,omitempty"` } From 3c06a8b69c1f7900419daa8884ed62dcaf24c6cb Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Mon, 1 Jun 2026 19:18:48 +0500 Subject: [PATCH 02/16] feat(runners): one-time registration token --- .dredd/hooks/helpers.go | 2 + api-docs.yml | 7 +- api/api_test.go | 1 + api/router.go | 3 +- api/runners.go | 36 ++-------- api/runners/runners.go | 1 + cli/cmd/root.go | 3 + db/Migration.go | 1 + db/Runner.go | 25 +++++-- db/bolt/global_runner.go | 6 -- db/bolt/global_runner_test.go | 46 ++++++++----- db/sql/global_runner.go | 14 ++-- db/sql/migrations/v2.18.7.err.sql | 2 + db/sql/migrations/v2.18.7.sql | 2 + pro/api/projects/runners.go | 5 +- services/export/Runner.go | 1 + services/project/restore.go | 1 + services/server/runner_svc.go | 105 +++++++++++++++++++++++++++++ services/server/runner_svc_test.go | 58 ++++++++++++++++ 19 files changed, 249 insertions(+), 70 deletions(-) create mode 100644 db/sql/migrations/v2.18.7.err.sql create mode 100644 db/sql/migrations/v2.18.7.sql create mode 100644 services/server/runner_svc.go create mode 100644 services/server/runner_svc_test.go diff --git a/.dredd/hooks/helpers.go b/.dredd/hooks/helpers.go index e4e83e622f..366c5a7004 100644 --- a/.dredd/hooks/helpers.go +++ b/.dredd/hooks/helpers.go @@ -314,6 +314,7 @@ func addIntegrationMatcher() *db.IntegrationMatcher { func addRunner() *db.Runner { runner, err := store.CreateRunner(db.Runner{ + Token: db.GenerateRunnerToken(), ProjectID: &userProject.ID, Name: "ITRN-" + getUUID(), Active: true, @@ -329,6 +330,7 @@ func addRunner() *db.Runner { func addGlobalRunner() *db.Runner { runner, err := store.CreateRunner(db.Runner{ + Token: db.GenerateRunnerToken(), ProjectID: nil, Name: "ITGRN-" + getUUID(), Active: true, diff --git a/api-docs.yml b/api-docs.yml index 5db73e4855..f9eba66fdf 100644 --- a/api-docs.yml +++ b/api-docs.yml @@ -1204,6 +1204,11 @@ definitions: type: string format: date-time x-nullable: true + registered: + type: boolean + registration_token: + type: string + x-nullable: true RunnerRequest: type: object @@ -1223,7 +1228,7 @@ definitions: type: array items: type: string - unregistered: + registered: type: boolean description: > When true, the runner is created without a token and stays inactive. diff --git a/api/api_test.go b/api/api_test.go index 19bc170117..2bd8b4f547 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -29,6 +29,7 @@ func TestApiPing(t *testing.T) { nil, nil, nil, + nil, ) r.ServeHTTP(rr, req) diff --git a/api/router.go b/api/router.go index 54b9687464..3e8062371c 100644 --- a/api/router.go +++ b/api/router.go @@ -93,6 +93,7 @@ func Route( accessKeyService server.AccessKeyService, environmentService server.EnvironmentService, subscriptionService pro_interfaces.SubscriptionService, + runnerService server.RunnerService, ) *mux.Router { projectController := &projects.ProjectController{ProjectService: projectService} @@ -108,7 +109,7 @@ func Route( userController := NewUserController(subscriptionService) usersController := NewUsersController(subscriptionService) subscriptionController := proApi.NewSubscriptionController(store, store, store, terraformStore) - projectRunnerController := proProjects.NewProjectRunnerController(subscriptionService) + projectRunnerController := proProjects.NewProjectRunnerController(subscriptionService, runnerService) taskController := projects.NewTaskController(ansibleTaskRepo) rolesController := proApi.NewRolesController(store) templateController := projects.NewTemplateController(store, store) diff --git a/api/runners.go b/api/runners.go index 5b3c16401f..9bbd89c22d 100644 --- a/api/runners.go +++ b/api/runners.go @@ -1,13 +1,11 @@ package api import ( - "bufio" - "bytes" "net/http" "github.com/semaphoreui/semaphore/api/helpers" "github.com/semaphoreui/semaphore/db" - "github.com/semaphoreui/semaphore/util" + "github.com/semaphoreui/semaphore/services/server" log "github.com/sirupsen/logrus" ) @@ -39,35 +37,9 @@ func addGlobalRunner(w http.ResponseWriter, r *http.Request) { runner.ProjectID = nil - var privateKey []byte + runnerService := helpers.GetFromContext(r, "runner_service").(server.RunnerService) - if runner.Unregistered { - // An unregistered runner has no token and cannot be active. Its keypair is - // generated by the runner itself when it registers via the CLI later on. - runner.Active = false - runner.PublicKey = nil - } else if runner.PublicKey == nil { - var b bytes.Buffer - privateKeyFile := bufio.NewWriter(&b) - - publicKey, err := util.GeneratePrivateKey(privateKeyFile) - if err != nil { - helpers.WriteError(w, err) - return - } - - err = privateKeyFile.Flush() - if err != nil { - helpers.WriteError(w, err) - return - } - - privateKey = b.Bytes() - - runner.PublicKey = &publicKey - } - - newRunner, err := helpers.Store(r).CreateRunner(runner) + newRunner, privateKey, err := runnerService.CreateRunner(runner) if err != nil { log.Warn("Runner is not created: " + err.Error()) @@ -78,7 +50,7 @@ func addGlobalRunner(w http.ResponseWriter, r *http.Request) { helpers.WriteJSON(w, http.StatusCreated, runnerWithToken{ Runner: newRunner, Token: newRunner.Token, - PrivateKey: string(privateKey), + PrivateKey: privateKey, }) } diff --git a/api/runners/runners.go b/api/runners/runners.go index a2490ebd1c..41e6f84b42 100644 --- a/api/runners/runners.go +++ b/api/runners/runners.go @@ -359,6 +359,7 @@ func RegisterRunner(w http.ResponseWriter, r *http.Request) { runner, err = helpers.Store(r).RegisterRunner(*register.RunnerID, register.PublicKey, register.Enabled) } else { runner, err = helpers.Store(r).CreateRunner(db.Runner{ + Token: db.GenerateRunnerToken(), Webhook: register.Webhook, Name: register.Name, Tags: register.Tags, diff --git a/cli/cmd/root.go b/cli/cmd/root.go index dbbafd97ef..4a70b4bc9d 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -98,6 +98,7 @@ func runService() { secretStorageService := server.NewSecretStorageService(store, store, accessKeyService, encryptionService) secretStorageSyncScheduler := server.NewSecretStorageSyncScheduler(store, secretStorageService) environmentService := server.NewEnvironmentService(store, encryptionService) + runnerService := server.NewRunnerService(store) subscriptionService := proServer.NewSubscriptionService(store, store, store, terraformStore) logWriteService := proServer.NewLogWriteService() @@ -208,6 +209,7 @@ func runService() { accessKeyService, environmentService, subscriptionService, + runnerService, ) route.Use(func(next http.Handler) http.Handler { @@ -217,6 +219,7 @@ func runService() { r = helpers.SetContextValue(r, "task_pool", &taskPool) r = helpers.SetContextValue(r, "log_writer", logWriteService) r = helpers.SetContextValue(r, "cluster_inspector", clusterInspector) + r = helpers.SetContextValue(r, "runner_service", runnerService) next.ServeHTTP(w, r) }) diff --git a/db/Migration.go b/db/Migration.go index 03a67e118c..75d1902ee0 100644 --- a/db/Migration.go +++ b/db/Migration.go @@ -126,6 +126,7 @@ func GetMigrations(dialect string) []Migration { {Version: "2.18.2"}, {Version: "2.18.4"}, {Version: "2.18.5"}, + {Version: "2.18.7"}, } return append(initScripts, commonScripts...) diff --git a/db/Runner.go b/db/Runner.go index 89a00324a6..f9f33f024d 100644 --- a/db/Runner.go +++ b/db/Runner.go @@ -1,8 +1,11 @@ package db import ( + "encoding/base64" "slices" "time" + + "github.com/gorilla/securecookie" ) type RunnerState string @@ -31,10 +34,19 @@ type Runner struct { PublicKey *string `db:"public_key" json:"-"` - // Unregistered is a transient flag (never persisted) used at creation time to - // request a runner without a token. Such a runner gets an empty token and must - // be registered later via `semaphore runner register --runner-id `. - Unregistered bool `db:"-" json:"unregistered"` + // Registered is a transient flag (never persisted) used at creation time to + // request a runner without an auth token. Such a runner gets a one-time, + // short-lived registration token instead and must be registered later by + // presenting that token to `semaphore runner register`. + Registered bool `db:"-" json:"registered"` + + // RegistrationTokenHash is the stored SHA-256 hash of the one-time registration + // token (the plaintext is never persisted). + RegistrationTokenHash *string `db:"registration_token" json:"-"` + // RegistrationToken is the transient plaintext registration token, returned to + // the caller only once at creation time. + RegistrationToken string `db:"-" json:"registration_token,omitempty"` + RegistrationTokenExpiresAt *time.Time `db:"registration_token_expires_at" json:"-"` } // IsRegistered reports whether the runner has been registered (has a token). @@ -42,6 +54,11 @@ func (r Runner) IsRegistered() bool { return r.Token != "" } +// GenerateRunnerToken creates a new runner authentication token. +func GenerateRunnerToken() string { + return base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32)) +} + // HasTag reports whether the runner is tagged with the given tag. func (r Runner) HasTag(tag string) bool { return slices.Contains(r.Tags, tag) diff --git a/db/bolt/global_runner.go b/db/bolt/global_runner.go index 5e328b7919..a5176d98ee 100644 --- a/db/bolt/global_runner.go +++ b/db/bolt/global_runner.go @@ -132,12 +132,6 @@ func (d *BoltDb) UpdateRunner(runner db.Runner) (err error) { } func (d *BoltDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) { - if runner.Unregistered { - runner.Token = "" - } else { - runner.Token = base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32)) - } - res, err := d.createObject(0, db.GlobalRunnerProps, runner) if err != nil { diff --git a/db/bolt/global_runner_test.go b/db/bolt/global_runner_test.go index d7362cf06f..f71a521033 100644 --- a/db/bolt/global_runner_test.go +++ b/db/bolt/global_runner_test.go @@ -2,16 +2,19 @@ package bolt import ( "testing" + "time" "github.com/semaphoreui/semaphore/db" + "github.com/semaphoreui/semaphore/pkg/tz" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_GetRunnerByToken_ReturnsGlobalRunnerWhenTokenExists(t *testing.T) { store := CreateTestStore() - testRunner, err := store.CreateRunner(db.Runner{}) - assert.NoError(t, err) + testRunner, err := store.CreateRunner(db.Runner{Token: db.GenerateRunnerToken()}) + require.NoError(t, err) _, err = store.GetRunnerByToken(testRunner.Token) assert.NoError(t, err) @@ -21,10 +24,10 @@ func Test_GetRunnerByToken_ReturnsRunnerWhenTokenExists(t *testing.T) { store := CreateTestStore() project, err := store.CreateProject(db.Project{}) - assert.NoError(t, err) + require.NoError(t, err) - testRunner, err := store.CreateRunner(db.Runner{ProjectID: &project.ID}) - assert.NoError(t, err) + testRunner, err := store.CreateRunner(db.Runner{ProjectID: &project.ID, Token: db.GenerateRunnerToken()}) + require.NoError(t, err) _, err = store.GetRunnerByToken(testRunner.Token) assert.NoError(t, err) @@ -43,34 +46,45 @@ func Test_GetGlobalRunner_ReturnsErrorWhenTryingGetProjectRunner(t *testing.T) { assert.ErrorIs(t, err, db.ErrNotFound) } -func Test_CreateRunner_UnregisteredHasEmptyToken(t *testing.T) { +// CreateRunner is a pure persister: it stores the registration-token hash and +// expiry it is given without altering them. Credential generation itself lives +// in the RunnerService. +func Test_CreateRunner_PersistsRegistrationTokenFields(t *testing.T) { store := CreateTestStore() - testRunner, err := store.CreateRunner(db.Runner{Unregistered: true}) - assert.NoError(t, err) + hash := db.HashRunnerRegistrationToken("plaintext-token") + expiresAt := tz.Now().Add(time.Hour) + + testRunner, err := store.CreateRunner(db.Runner{ + RegistrationTokenHash: &hash, + RegistrationTokenExpiresAt: &expiresAt, + }) + require.NoError(t, err) assert.Empty(t, testRunner.Token) stored, err := store.GetGlobalRunner(testRunner.ID) - assert.NoError(t, err) + require.NoError(t, err) assert.Empty(t, stored.Token) - assert.False(t, stored.IsRegistered()) + require.NotNil(t, stored.RegistrationTokenHash) + assert.Equal(t, hash, *stored.RegistrationTokenHash) + assert.NotNil(t, stored.RegistrationTokenExpiresAt) } func Test_RegisterRunner_SetsTokenAndActivates(t *testing.T) { store := CreateTestStore() - testRunner, err := store.CreateRunner(db.Runner{Unregistered: true}) - assert.NoError(t, err) + testRunner, err := store.CreateRunner(db.Runner{}) + require.NoError(t, err) publicKey := "test-public-key" registered, err := store.RegisterRunner(testRunner.ID, &publicKey, true) - assert.NoError(t, err) + require.NoError(t, err) assert.NotEmpty(t, registered.Token) assert.True(t, registered.Active) assert.Equal(t, publicKey, *registered.PublicKey) stored, err := store.GetGlobalRunner(testRunner.ID) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, registered.Token, stored.Token) assert.True(t, stored.Active) } @@ -78,8 +92,8 @@ func Test_RegisterRunner_SetsTokenAndActivates(t *testing.T) { func Test_RegisterRunner_FailsWhenAlreadyRegistered(t *testing.T) { store := CreateTestStore() - testRunner, err := store.CreateRunner(db.Runner{}) - assert.NoError(t, err) + testRunner, err := store.CreateRunner(db.Runner{Token: db.GenerateRunnerToken()}) + require.NoError(t, err) assert.NotEmpty(t, testRunner.Token) _, err = store.RegisterRunner(testRunner.ID, nil, true) diff --git a/db/sql/global_runner.go b/db/sql/global_runner.go index 9ab812a117..1daee98d03 100644 --- a/db/sql/global_runner.go +++ b/db/sql/global_runner.go @@ -205,22 +205,19 @@ func (d *SqlDb) RegisterRunner(runnerID int, publicKey *string, active bool) (ru } func (d *SqlDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) { - token := "" - if !runner.Unregistered { - token = base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32)) - } - insertID, err := d.insert( "id", - "insert into `runner` (project_id, token, webhook, max_parallel_tasks, `name`, `active`, `is_default`, public_key) values (?, ?, ?, ?, ?, ?, ?, ?)", + "insert into `runner` (project_id, token, webhook, max_parallel_tasks, `name`, `active`, `is_default`, public_key, registration_token, registration_token_expires_at) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", runner.ProjectID, - token, + runner.Token, runner.Webhook, runner.MaxParallelTasks, runner.Name, runner.Active, runner.IsDefault, - runner.PublicKey) + runner.PublicKey, + runner.RegistrationTokenHash, + runner.RegistrationTokenExpiresAt) if err != nil { return @@ -228,7 +225,6 @@ func (d *SqlDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) newRunner = runner newRunner.ID = insertID - newRunner.Token = token newRunner.Tags = normalizeTags(runner.Tags) if err = d.replaceRunnerTags(newRunner.ID, newRunner.Tags); err != nil { diff --git a/db/sql/migrations/v2.18.7.err.sql b/db/sql/migrations/v2.18.7.err.sql new file mode 100644 index 0000000000..ade763fc1a --- /dev/null +++ b/db/sql/migrations/v2.18.7.err.sql @@ -0,0 +1,2 @@ +alter table `runner` drop column `registration_token`; +alter table `runner` drop column `registration_token_expires_at`; diff --git a/db/sql/migrations/v2.18.7.sql b/db/sql/migrations/v2.18.7.sql new file mode 100644 index 0000000000..4396a9b1c6 --- /dev/null +++ b/db/sql/migrations/v2.18.7.sql @@ -0,0 +1,2 @@ +alter table `runner` add column `registration_token` varchar(255); +alter table `runner` add column `registration_token_expires_at` datetime; diff --git a/pro/api/projects/runners.go b/pro/api/projects/runners.go index 4b2e2a5e3f..1ea9052f17 100644 --- a/pro/api/projects/runners.go +++ b/pro/api/projects/runners.go @@ -8,7 +8,10 @@ import ( ) // NewProjectRunnerController creates a new ProjectRunnerController instance. -func NewProjectRunnerController(subscriptionService pro_interfaces.SubscriptionService) pro_interfaces.ProjectRunnerController { +func NewProjectRunnerController( + subscriptionService pro_interfaces.SubscriptionService, + runnerService server.RunnerService, +) pro_interfaces.ProjectRunnerController { return &ProjectRunnerControllerImpl{} } diff --git a/services/export/Runner.go b/services/export/Runner.go index e30f53e15e..a48d384c6a 100644 --- a/services/export/Runner.go +++ b/services/export/Runner.go @@ -34,6 +34,7 @@ func (e *RunnerExporter) restoreValue(val EntityObject[db.Runner], store db.Stor return err } + old.Token = db.GenerateRunnerToken() newObj, err := store.CreateRunner(old) if err != nil { return err diff --git a/services/project/restore.go b/services/project/restore.go index 2d6ed034ed..c6adf2b06b 100644 --- a/services/project/restore.go +++ b/services/project/restore.go @@ -462,6 +462,7 @@ func (e BackupRunner) Verify(backup *BackupFormat) error { func (e BackupRunner) Restore(store db.Store, b *BackupDB) error { runner := e.Runner runner.ProjectID = &b.meta.ID + runner.Token = "" // Unregistered runner newRunner, err := store.CreateRunner(runner) if err != nil { return err diff --git a/services/server/runner_svc.go b/services/server/runner_svc.go new file mode 100644 index 0000000000..03bcf80b5e --- /dev/null +++ b/services/server/runner_svc.go @@ -0,0 +1,105 @@ +package server + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "time" + + "github.com/gorilla/securecookie" + "github.com/semaphoreui/semaphore/db" + "github.com/semaphoreui/semaphore/pkg/tz" + "github.com/semaphoreui/semaphore/util" +) + +// runnerRegistrationTokenTTL is how long a one-time registration token issued for +// an unregistered runner stays valid. +const runnerRegistrationTokenTTL = time.Hour + +// generateRunnerRegistrationToken creates a new one-time registration token and +// returns the plaintext token (handed to the caller once) together with its hash +// (stored in the database, never the plaintext). +func generateRunnerRegistrationToken() (token string, hash string) { + token = base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32)) + hash = HashRunnerRegistrationToken(token) + return +} + +// HashRunnerRegistrationToken hashes a registration token for storage and lookup. +// It is deterministic (SHA-256) so a runner can be found by the token it presents. +func HashRunnerRegistrationToken(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} + +// RunnerService owns the creation of runners for both global and project scopes: +// it decides and generates a runner's credentials (auth token or one-time +// registration token) and, when needed, its key pair, before persisting it. +type RunnerService interface { + // CreateRunner generates the runner's credentials and persists it. The returned + // privateKey is non-empty only when the service generated a key pair for the + // runner (and must be handed to the caller exactly once). + CreateRunner(runner db.Runner) (newRunner db.Runner, privateKey string, err error) +} + +type RunnerServiceImpl struct { + runnerRepo db.RunnerManager +} + +func NewRunnerService(runnerRepo db.RunnerManager) RunnerService { + return &RunnerServiceImpl{ + runnerRepo: runnerRepo, + } +} + +func (s *RunnerServiceImpl) CreateRunner(runner db.Runner) (newRunner db.Runner, privateKey string, err error) { + if runner.Registered { + runner.Token = db.GenerateRunnerToken() + + if runner.PublicKey == nil { + privateKey, err = s.generateKeyPair(&runner) + if err != nil { + return + } + } + } else { + // An unregistered runner has no auth token and cannot be active. It is given + // a one-time, short-lived registration token instead: only its hash and + // expiry are persisted, the plaintext is returned to the caller once. Its + // key pair is generated by the runner itself when it registers later. + runner.Token = "" + runner.Active = false + runner.PublicKey = nil + + token, hash := generateRunnerRegistrationToken() + expiresAt := tz.Now().Add(runnerRegistrationTokenTTL) + runner.RegistrationToken = token + runner.RegistrationTokenHash = &hash + runner.RegistrationTokenExpiresAt = &expiresAt + } + + newRunner, err = s.runnerRepo.CreateRunner(runner) + return +} + +// generateKeyPair generates an RSA key pair for the runner, sets its public key +// and returns the PEM-encoded private key. +func (s *RunnerServiceImpl) generateKeyPair(runner *db.Runner) (privateKey string, err error) { + var b bytes.Buffer + w := bufio.NewWriter(&b) + + publicKey, err := util.GeneratePrivateKey(w) + if err != nil { + return + } + + if err = w.Flush(); err != nil { + return + } + + runner.PublicKey = &publicKey + privateKey = b.String() + return +} diff --git a/services/server/runner_svc_test.go b/services/server/runner_svc_test.go new file mode 100644 index 0000000000..3b0a5a699c --- /dev/null +++ b/services/server/runner_svc_test.go @@ -0,0 +1,58 @@ +package server + +import ( + "testing" + + "github.com/semaphoreui/semaphore/db" + "github.com/semaphoreui/semaphore/db/bolt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRunnerService_CreateRunner_Registered(t *testing.T) { + svc := NewRunnerService(bolt.CreateTestStore()) + + runner, privateKey, err := svc.CreateRunner(db.Runner{Name: "r1", Registered: true}) + require.NoError(t, err) + + // A normal runner gets an auth token and a server-generated key pair. + assert.NotEmpty(t, runner.Token) + assert.True(t, runner.IsRegistered()) + assert.NotEmpty(t, privateKey) + assert.NotNil(t, runner.PublicKey) + assert.Empty(t, runner.RegistrationToken) + assert.Nil(t, runner.RegistrationTokenHash) +} + +func TestRunnerService_CreateRunner_RegisteredWithProvidedPublicKey(t *testing.T) { + svc := NewRunnerService(bolt.CreateTestStore()) + + pub := "provided-public-key" + runner, privateKey, err := svc.CreateRunner(db.Runner{PublicKey: &pub}) + require.NoError(t, err) + + // When the caller provides a public key, the service does not generate one. + assert.NotEmpty(t, runner.Token) + assert.Empty(t, privateKey) + require.NotNil(t, runner.PublicKey) + assert.Equal(t, pub, *runner.PublicKey) +} + +func TestRunnerService_CreateRunner_Unregistered(t *testing.T) { + svc := NewRunnerService(bolt.CreateTestStore()) + + runner, privateKey, err := svc.CreateRunner(db.Runner{Registered: false, Active: true}) + require.NoError(t, err) + + // An unregistered runner has no auth token, is forced inactive, gets no key + // pair, and receives a one-time registration token instead. + assert.Empty(t, runner.Token) + assert.False(t, runner.IsRegistered()) + assert.False(t, runner.Active) + assert.Empty(t, privateKey) + assert.Nil(t, runner.PublicKey) + assert.NotEmpty(t, runner.RegistrationToken) + require.NotNil(t, runner.RegistrationTokenHash) + //assert.Equal(t, db.HashRunnerRegistrationToken(runner.RegistrationToken), *runner.RegistrationTokenHash) + assert.NotNil(t, runner.RegistrationTokenExpiresAt) +} From d29563498dc69ab90743180c61eea1e86ca99b54 Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Mon, 1 Jun 2026 19:22:11 +0500 Subject: [PATCH 03/16] test(runners): pass tests --- db/bolt/global_runner_test.go | 3 ++- services/server/runner_svc_test.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/db/bolt/global_runner_test.go b/db/bolt/global_runner_test.go index f71a521033..752461f5f3 100644 --- a/db/bolt/global_runner_test.go +++ b/db/bolt/global_runner_test.go @@ -6,6 +6,7 @@ import ( "github.com/semaphoreui/semaphore/db" "github.com/semaphoreui/semaphore/pkg/tz" + "github.com/semaphoreui/semaphore/services/server" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -52,7 +53,7 @@ func Test_GetGlobalRunner_ReturnsErrorWhenTryingGetProjectRunner(t *testing.T) { func Test_CreateRunner_PersistsRegistrationTokenFields(t *testing.T) { store := CreateTestStore() - hash := db.HashRunnerRegistrationToken("plaintext-token") + hash := server.HashRunnerRegistrationToken("plaintext-token") expiresAt := tz.Now().Add(time.Hour) testRunner, err := store.CreateRunner(db.Runner{ diff --git a/services/server/runner_svc_test.go b/services/server/runner_svc_test.go index 3b0a5a699c..d353665964 100644 --- a/services/server/runner_svc_test.go +++ b/services/server/runner_svc_test.go @@ -28,7 +28,7 @@ func TestRunnerService_CreateRunner_RegisteredWithProvidedPublicKey(t *testing.T svc := NewRunnerService(bolt.CreateTestStore()) pub := "provided-public-key" - runner, privateKey, err := svc.CreateRunner(db.Runner{PublicKey: &pub}) + runner, privateKey, err := svc.CreateRunner(db.Runner{PublicKey: &pub, Registered: true}) require.NoError(t, err) // When the caller provides a public key, the service does not generate one. @@ -53,6 +53,6 @@ func TestRunnerService_CreateRunner_Unregistered(t *testing.T) { assert.Nil(t, runner.PublicKey) assert.NotEmpty(t, runner.RegistrationToken) require.NotNil(t, runner.RegistrationTokenHash) - //assert.Equal(t, db.HashRunnerRegistrationToken(runner.RegistrationToken), *runner.RegistrationTokenHash) + assert.Equal(t, HashRunnerRegistrationToken(runner.RegistrationToken), *runner.RegistrationTokenHash) assert.NotNil(t, runner.RegistrationTokenExpiresAt) } From de47599ba633c0d55231d8d403dfaffd3e0db01d Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Mon, 1 Jun 2026 20:07:27 +0500 Subject: [PATCH 04/16] ci: use custom pro branch --- .github/workflows/dev.yml | 2 +- .github/workflows/pro_selfhosted_beta.yml | 2 +- .github/workflows/pro_selfhosted_release.yml | 2 +- deployment/docker/runner/Dockerfile | 2 +- deployment/docker/server/Dockerfile | 2 +- web/src/components/RunnerForm.vue | 39 ++++- web/src/lang/en.js | 10 ++ web/src/views/Runners.vue | 166 ++++++++++++++++++- 8 files changed, 211 insertions(+), 14 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 2f06bd070e..a662342898 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -39,7 +39,7 @@ jobs: - name: Add PRO implementation run: | - git clone -b main https://${{ secrets.GH_TOKEN }}@github.com/semaphoreui/semaphorepro-module.git pro_impl + git clone -b feat/unregistered_runners https://${{ secrets.GH_TOKEN }}@github.com/semaphoreui/semaphorepro-module.git pro_impl go work init . ./pro_impl - name: Run build diff --git a/.github/workflows/pro_selfhosted_beta.yml b/.github/workflows/pro_selfhosted_beta.yml index fde9d20245..e906808599 100644 --- a/.github/workflows/pro_selfhosted_beta.yml +++ b/.github/workflows/pro_selfhosted_beta.yml @@ -39,7 +39,7 @@ jobs: - name: Add PRO implementation run: | - git clone -b main https://${{ secrets.GH_TOKEN }}@github.com/semaphoreui/semaphorepro-module.git pro_impl + git clone -b feat/unregistered_runners https://${{ secrets.GH_TOKEN }}@github.com/semaphoreui/semaphorepro-module.git pro_impl go work init . ./pro_impl - name: Install deps diff --git a/.github/workflows/pro_selfhosted_release.yml b/.github/workflows/pro_selfhosted_release.yml index 08ff52cc37..b7971fd191 100644 --- a/.github/workflows/pro_selfhosted_release.yml +++ b/.github/workflows/pro_selfhosted_release.yml @@ -38,7 +38,7 @@ jobs: - name: Add PRO implementation run: | - git clone -b main https://${{ secrets.GH_TOKEN }}@github.com/semaphoreui/semaphorepro-module.git pro_impl + git clone -b feat/unregistered_runners https://${{ secrets.GH_TOKEN }}@github.com/semaphoreui/semaphorepro-module.git pro_impl go work init . ./pro_impl - name: Install deps diff --git a/deployment/docker/runner/Dockerfile b/deployment/docker/runner/Dockerfile index ea14a20d19..4185a17c46 100644 --- a/deployment/docker/runner/Dockerfile +++ b/deployment/docker/runner/Dockerfile @@ -19,7 +19,7 @@ ARG TARGETARCH ARG GH_TOKEN RUN if [ -n "$APP_BUILD_TYPE" ]; then \ - git clone -b main https://${GH_TOKEN}@github.com/semaphoreui/semaphorepro-module.git pro_impl && \ + git clone -b feat/unregistered_runners https://${GH_TOKEN}@github.com/semaphoreui/semaphorepro-module.git pro_impl && \ go work init . ./pro_impl; \ fi diff --git a/deployment/docker/server/Dockerfile b/deployment/docker/server/Dockerfile index fe6db76cd7..7e534219e0 100644 --- a/deployment/docker/server/Dockerfile +++ b/deployment/docker/server/Dockerfile @@ -19,7 +19,7 @@ ARG TARGETARCH ARG GH_TOKEN RUN if [ -n "$APP_BUILD_TYPE" ]; then \ - git clone -b main https://${GH_TOKEN}@github.com/semaphoreui/semaphorepro-module.git pro_impl && \ + git clone -b feat/unregistered_runners https://${GH_TOKEN}@github.com/semaphoreui/semaphorepro-module.git pro_impl && \ go work init . ./pro_impl; \ fi diff --git a/web/src/components/RunnerForm.vue b/web/src/components/RunnerForm.vue index 98e95ca885..2b591562d1 100644 --- a/web/src/components/RunnerForm.vue +++ b/web/src/components/RunnerForm.vue @@ -47,12 +47,32 @@ - - - + + + + + + + + + + + + + + + {{ $t('unregisteredRunnerHint') }} + Enterprise edition.', ha_only_enterprise: 'High availability are only available in Enterprise edition.', diff --git a/web/src/views/Runners.vue b/web/src/views/Runners.vue index f2a1d1823f..878e8362b0 100644 --- a/web/src/views/Runners.vue +++ b/web/src/views/Runners.vue @@ -31,11 +31,131 @@ :max-width="600" v-model="newRunnerTokenDialog" :save-button-text="null" - :title="$t('newRunnerToken')" + :title="isUnregisteredRunner ? $t('runnerRegistrationToken') : $t('newRunnerToken')" hide-buttons >