feat(runners): allow to create unregistered runners for terraform usage

api-docs.yml

@@ -1223,6 +1223,14 @@ definitions:
         type: array
         items:
           type: string
+      unregistered:
+        type: boolean
+        description: >
+          When true, the runner is created without a token and stays inactive.
+          The token is not stored; the runner must be registered later via
+          `semaphore runner register --runner-id <id>`. Useful for tools like
+          Terraform that create the runner up front and pass its ID to the
+          runner host via cloud-init.
 
   RunnerWithToken:
     allOf:

api/runners.go

@@ -41,7 +41,12 @@ func addGlobalRunner(w http.ResponseWriter, r *http.Request) {
 
 	var privateKey []byte
 
-	if runner.PublicKey == nil {
+	if runner.Unregistered {
+		// An unregistered runner has no token and cannot be active. Its keypair is
+		// generated by the runner itself when it registers via the CLI later on.
+		runner.Active = false
+		runner.PublicKey = nil
+	} else if runner.PublicKey == nil {
 		var b bytes.Buffer
 		privateKeyFile := bufio.NewWriter(&b)
 
@@ -118,6 +123,11 @@ func updateGlobalRunner(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if runner.Active && !oldRunner.IsRegistered() {
+		helpers.WriteErrorStatus(w, "Unregistered runner cannot be activated", http.StatusBadRequest)
+		return
+	}
+
 	store := helpers.Store(r)
 
 	runner.ID = oldRunner.ID
@@ -188,6 +198,11 @@ func setGlobalRunnerActive(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if body.Active && !runner.IsRegistered() {
+		helpers.WriteErrorStatus(w, "Unregistered runner cannot be activated", http.StatusBadRequest)
+		return
+	}
+
 	runner.Active = body.Active
 
 	err := store.UpdateRunner(*runner)

api/runners/runners.go

@@ -351,15 +351,23 @@ func RegisterRunner(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	runner, err := helpers.Store(r).CreateRunner(db.Runner{
-		Webhook:          register.Webhook,
-		Name:             register.Name,
-		Tags:             register.Tags,
-		MaxParallelTasks: register.MaxParallelTasks,
-		Active:           register.Enabled,
-		PublicKey:        register.PublicKey,
-		ProjectID:        register.ProjectID,
-	})
+	var runner db.Runner
+	var err error
+
+	if register.RunnerID != nil {
+		// Register a previously created tokenless runner by its ID.
+		runner, err = helpers.Store(r).RegisterRunner(*register.RunnerID, register.PublicKey, register.Enabled)
+	} else {
+		runner, err = helpers.Store(r).CreateRunner(db.Runner{
+			Webhook:          register.Webhook,
+			Name:             register.Name,
+			Tags:             register.Tags,
+			MaxParallelTasks: register.MaxParallelTasks,
+			Active:           register.Enabled,
+			PublicKey:        register.PublicKey,
+			ProjectID:        register.ProjectID,
+		})
+	}
 
 	if err != nil {
 

cli/cmd/runner_register.go

@@ -13,6 +13,7 @@ var runnerRegisterArgs struct {
 	stdinRegistrationToken    bool
 	registrationTokenFilePath string
 	projectID                 int
+	runnerID                  int
 	name                      string
 	tags                      []string
 	webhook                   string
@@ -30,6 +31,7 @@ func init() {
 	runnerRegisterCmd.PersistentFlags().StringVar(&runnerRegisterArgs.registrationTokenFilePath, "registration-token-file", "", "Read registration token from a file")
 	runnerRegisterCmd.PersistentFlags().BoolVar(&runnerRegisterArgs.enabled, "enabled", true, "Enable or disable the runner on the server")
 	runnerRegisterCmd.PersistentFlags().IntVar(&runnerRegisterArgs.projectID, "project-id", 0, "Project ID for project-level runner (global runner if not provided)")
+	runnerRegisterCmd.PersistentFlags().IntVar(&runnerRegisterArgs.runnerID, "runner-id", 0, "ID of an existing unregistered runner to register (created beforehand, e.g. via Terraform)")
 	runnerCmd.AddCommand(runnerRegisterCmd)
 }
 
@@ -100,6 +102,10 @@ func registerRunner(cmd *cobra.Command) {
 		util.Config.Runner.ProjectID = &runnerRegisterArgs.projectID
 	}
 
+	if runnerRegisterArgs.runnerID > 0 {
+		util.Config.Runner.RunnerID = &runnerRegisterArgs.runnerID
+	}
+
 	taskPool := createRunnerJobPool()
 
 	err := taskPool.Register(configFile)

config.schema.yaml

@@ -467,3 +467,6 @@ $defs:
       enabled:            { type: boolean }
       project_id:
         type: [integer, "null"]
+      runner_id:
+        type: [integer, "null"]
+        description: ID of an existing unregistered runner to register instead of creating a new one.

db/Runner.go

@@ -30,6 +30,16 @@ type Runner struct {
 	CleaningRequested *time.Time `db:"cleaning_requested" json:"cleaning_requested"`
 
 	PublicKey *string `db:"public_key" json:"-"`
+
+	// Unregistered is a transient flag (never persisted) used at creation time to
+	// request a runner without a token. Such a runner gets an empty token and must
+	// be registered later via `semaphore runner register --runner-id <id>`.
+	Unregistered bool `db:"-" json:"unregistered"`
+}
+
+// IsRegistered reports whether the runner has been registered (has a token).
+func (r Runner) IsRegistered() bool {
+	return r.Token != ""
 }
 
 // HasTag reports whether the runner is tagged with the given tag.

db/Store.go

@@ -458,6 +458,9 @@ type RunnerManager interface {
 	DeleteGlobalRunner(runnerID int) error
 	UpdateRunner(runner Runner) error
 	CreateRunner(runner Runner) (Runner, error)
+	// RegisterRunner finalizes a previously created tokenless ("unregistered")
+	// runner by generating its token, storing its public key and activating it.
+	RegisterRunner(runnerID int, publicKey *string, active bool) (Runner, error)
 	TouchRunner(runner Runner) (err error)
 	ClearRunnerCache(runner Runner) (err error)
 	GetRunnerTags(projectID int) ([]RunnerTag, error)

db/bolt/global_runner.go

@@ -2,6 +2,7 @@ package bolt
 
 import (
 	"encoding/base64"
+	"fmt"
 
 	"github.com/gorilla/securecookie"
 	"github.com/semaphoreui/semaphore/db"
@@ -131,7 +132,11 @@ func (d *BoltDb) UpdateRunner(runner db.Runner) (err error) {
 }
 
 func (d *BoltDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) {
-	runner.Token = base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32))
+	if runner.Unregistered {
+		runner.Token = ""
+	} else {
+		runner.Token = base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32))
+	}
 
 	res, err := d.createObject(0, db.GlobalRunnerProps, runner)
 
@@ -141,3 +146,24 @@ func (d *BoltDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error)
 	newRunner = res.(db.Runner)
 	return
 }
+
+func (d *BoltDb) RegisterRunner(runnerID int, publicKey *string, active bool) (runner db.Runner, err error) {
+	err = d.db.Update(func(tx *bbolt.Tx) error {
+		err = d.getObjectTx(tx, 0, db.GlobalRunnerProps, intObjectID(runnerID), &runner)
+		if err != nil {
+			return err
+		}
+
+		if runner.IsRegistered() {
+			return fmt.Errorf("runner is already registered")
+		}
+
+		runner.Token = base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32))
+		runner.PublicKey = publicKey
+		runner.Active = active
+
+		return d.updateObjectTx(tx, 0, db.GlobalRunnerProps, runner)
+	})
+
+	return
+}

db/bolt/global_runner_test.go

@@ -42,3 +42,46 @@ func Test_GetGlobalRunner_ReturnsErrorWhenTryingGetProjectRunner(t *testing.T) {
 	_, err = store.GetGlobalRunner(testRunner.ID)
 	assert.ErrorIs(t, err, db.ErrNotFound)
 }
+
+func Test_CreateRunner_UnregisteredHasEmptyToken(t *testing.T) {
+	store := CreateTestStore()
+
+	testRunner, err := store.CreateRunner(db.Runner{Unregistered: true})
+	assert.NoError(t, err)
+	assert.Empty(t, testRunner.Token)
+
+	stored, err := store.GetGlobalRunner(testRunner.ID)
+	assert.NoError(t, err)
+	assert.Empty(t, stored.Token)
+	assert.False(t, stored.IsRegistered())
+}
+
+func Test_RegisterRunner_SetsTokenAndActivates(t *testing.T) {
+	store := CreateTestStore()
+
+	testRunner, err := store.CreateRunner(db.Runner{Unregistered: true})
+	assert.NoError(t, err)
+
+	publicKey := "test-public-key"
+	registered, err := store.RegisterRunner(testRunner.ID, &publicKey, true)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, registered.Token)
+	assert.True(t, registered.Active)
+	assert.Equal(t, publicKey, *registered.PublicKey)
+
+	stored, err := store.GetGlobalRunner(testRunner.ID)
+	assert.NoError(t, err)
+	assert.Equal(t, registered.Token, stored.Token)
+	assert.True(t, stored.Active)
+}
+
+func Test_RegisterRunner_FailsWhenAlreadyRegistered(t *testing.T) {
+	store := CreateTestStore()
+
+	testRunner, err := store.CreateRunner(db.Runner{})
+	assert.NoError(t, err)
+	assert.NotEmpty(t, testRunner.Token)
+
+	_, err = store.RegisterRunner(testRunner.ID, nil, true)
+	assert.Error(t, err)
+}

db/sql/global_runner.go

@@ -172,9 +172,44 @@ func (d *SqlDb) UpdateRunner(runner db.Runner) (err error) {
 	return
 }
 
-func (d *SqlDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) {
+func (d *SqlDb) RegisterRunner(runnerID int, publicKey *string, active bool) (runner db.Runner, err error) {
+	err = d.getObject(0, db.GlobalRunnerProps, runnerID, &runner)
+	if err != nil {
+		return
+	}
+
+	if runner.IsRegistered() {
+		err = fmt.Errorf("runner is already registered")
+		return
+	}
+
 	token := base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32))
 
+	_, err = d.exec(
+		"update `runner` set `token`=?, `public_key`=?, `active`=? where id=?",
+		token,
+		publicKey,
+		active,
+		runnerID)
+
+	if err != nil {
+		return
+	}
+
+	runner.Token = token
+	runner.PublicKey = publicKey
+	runner.Active = active
+
+	err = d.loadRunnerTagsSingle(&runner)
+	return
+}
+
+func (d *SqlDb) CreateRunner(runner db.Runner) (newRunner db.Runner, err error) {
+	token := ""
+	if !runner.Unregistered {
+		token = base64.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32))
+	}
+
 	insertID, err := d.insert(
 		"id",
 		"insert into `runner` (project_id, token, webhook, max_parallel_tasks, `name`, `active`, `is_default`, public_key) values (?, ?, ?, ?, ?, ?, ?, ?)",

services/runners/job_pool.go

@@ -397,6 +397,7 @@ func (p *JobPool) tryRegisterRunner(configFilePath *string) (ok bool) {
 
 	jsonBytes, err := json.Marshal(RunnerRegistration{
 		RegistrationToken: util.Config.Runner.RegistrationToken,
+		RunnerID:          util.Config.Runner.RunnerID,
 		Webhook:           util.Config.Runner.Webhook,
 		Name:              util.Config.Runner.Name,
 		Tags:              util.Config.Runner.Tags,

services/runners/types.go

@@ -56,7 +56,11 @@ type JobProgress struct {
 }
 
 type RunnerRegistration struct {
-	RegistrationToken string   `json:"registration_token" binding:"required"`
+	RegistrationToken string `json:"registration_token" binding:"required"`
+	// RunnerID, when set, registers a previously created tokenless ("unregistered")
+	// runner instead of creating a new one. Used by the Terraform provider, which
+	// creates the runner up front and passes its ID to the runner via cloud-init.
+	RunnerID          *int     `json:"runner_id,omitempty"`
 	Webhook           string   `json:"webhook,omitempty"`
 	Name              string   `json:"name,omitempty"`
 	Tags              []string `json:"tags,omitempty"`

util/config.go

@@ -150,6 +150,10 @@ type RunnerConfig struct {
 	MaxParallelTasks int      `json:"max_parallel_tasks,omitempty" default:"1" env:"SEMAPHORE_RUNNER_MAX_PARALLEL_TASKS"`
 	ProjectID        *int     `json:"project_id,omitempty" env:"SEMAPHORE_RUNNER_PROJECT_ID"`
 
+	// RunnerID, when set, registers a previously created tokenless ("unregistered")
+	// runner with this ID instead of creating a new runner on the server.
+	RunnerID *int `json:"runner_id,omitempty" env:"SEMAPHORE_RUNNER_ID"`
+
 	Connection *RunnerConnectionConfig `json:"connection,omitempty"`
 }