config

type AzureConfig struct {
	Name           string     `yaml:"-"`
	DefaultRegion  string     `yaml:"default_region"`
	SubscriptionID string     `yaml:"subscription_id"`
	ClientID       string     `yaml:"client_id"`
	ClientSecret   string     `yaml:"client_secret"`
	TenantID       string     `yaml:"tenant_id"`
	UseSpot        bool       `yaml:"use_spot" default:"true"`
	Username       string     `yaml:"username" default:"ubuntu"` // Default username for the VM, using OVH default
	SSHPublicKey   string     `yaml:"ssh_public_key" default:"~/.ssh/id_rsa.pub"`
	Image          AzureImage `yaml:"image"`
	// Image booted for workers recruited on a has_gpu=true flavor
	// (Standard_N* on Azure). Default: Microsoft's "Ubuntu HPC 22.04"
	// image — free on the marketplace, ships NVIDIA drivers + CUDA
	// runtime + nvidia-container-toolkit + docker preconfigured for
	// `--gpus all`. cloud-init then just installs scitq-client as
	// usual. Override per-field to point at a different image
	// (e.g. NVIDIA's own GPU-Optimized VMI or a custom baked one).
	GPUImage            AzureGPUImage     `yaml:"gpu_image"`
	Quotas              map[string]Quota  `yaml:"quotas"` // key: region
	Regions             []string          `yaml:"regions"`
	UpdatePeriodicity   string            `yaml:"update_periodicity"` // Update periodicity in minutes
	LocalWorkspaceRoots map[string]string `yaml:"local_workspaces"`   // key: region (or "*" wildcard); value: workspace URI for that region
	LocalResourceRoots  map[string]string `yaml:"local_resources"`    // key: region (or "*" wildcard); value: resource-root URI for that region
	// Regex lists applied at flavor-sync time. Logic: include first
	// (default = include all), then exclude (default = exclude none).
	// A flavor is synced into the catalog iff it matches at least one
	// include pattern (or include is empty) AND matches no exclude
	// pattern. Lets operators take entire VM families (e.g.
	// confidential `_cc_v\d+$`) out of recruitment without having to
	// disable each flavor individually after every Azure catalog update.
	FlavorIncludePatterns []string `yaml:"flavor_include_patterns"`
	FlavorExcludePatterns []string `yaml:"flavor_exclude_patterns"`
}

type AzureGPUImage struct {
	Publisher string `yaml:"publisher" default:"microsoft-dsvm"`
	Offer     string `yaml:"offer" default:"ubuntu-hpc"`
	Sku       string `yaml:"sku" default:"2204"`
	Version   string `yaml:"version" default:"latest"`
}

type AzureImage struct {
	Publisher string `yaml:"publisher" default:"Canonical"`
	Offer     string `yaml:"offer" default:"UbuntuServer"`
	Sku       string `yaml:"sku" default:"24.04-LTS"`
	Version   string `yaml:"version" default:"latest"`
}

type Config struct {
	// Scitq contains configuration parameters specific to the scitq server.
	Scitq struct {
		// Port is the TCP port on which the scitq server listens for gRPC incoming connections.
		Port int `yaml:"port" default:"50051"`

		// DBURL is the database connection string used by scitq to connect to its PostgreSQL database.
		// It should include the username, password, host, database name, and SSL mode.
		DBURL string `yaml:"db_url" default:"postgres://localhost/scitq2?sslmode=disable"`

		// MaxDBConcurrency limits the maximum number of concurrent database connections.
		MaxDBConcurrency int `yaml:"max_db_concurrency" default:"50"`

		// LogLevel sets the verbosity level of logging output.
		// Common values include "debug", "info", "warn", and "error".
		LogLevel string `yaml:"log_level" default:"info"`

		// LogRoot specifies the root directory where remote task stdout/stderr files are stored.
		LogRoot string `yaml:"log_root" default:"/var/lib/scitq/tasks"`

		// ScriptRoot is the directory where (python) server-side scripts are located.
		// Scripts run by scitq are expected to be found here.
		ScriptRoot string `yaml:"script_root" default:"/var/lib/scitq/scripts"`

		// ScriptVenv specifies the path to the Python virtual environment used to run scripts.
		// This isolates script dependencies from the system Python environment.
		// Python venv creation and DSL installation in the venv is managed automatically by server.
		ScriptVenv string `yaml:"script_venv" default:"/var/lib/scitq/python"`

		// ScriptRunnerUser is the system user account under which scripts are executed.
		// Running scripts as a non-privileged user enhances security.
		ScriptRunnerUser string `yaml:"script_runner_user" default:"nobody"`

		// ClientBinaryPath is the filesystem path to the scitq client binary.
		// This is used for automated client installation.
		ClientBinaryPath string `yaml:"client_binary_path" default:"/usr/local/bin/scitq-client"`

		// CliBinaryPath is the filesystem path to the scitq CLI binary. Served
		// from `/scitq-cli` so worker bootstrap scripts (and any task using
		// `task_spec.scitq_auth: true`) can keep their CLI in sync with the
		// rest of the deployment. Must be a fully-static build
		// (`CGO_ENABLED=0`) so it runs inside alpine/distroless containers
		// when bind-mounted — see `make build-cli`.
		CliBinaryPath string `yaml:"cli_binary_path" default:"/usr/local/bin/scitq"`

		// ClientDownloadToken is a secret token used to authorize client binary downloads.
		// If not set, a random token is generated at startup.
		ClientDownloadToken string `yaml:"client_download_token"`

		// CertificateKey is the path or content of the TLS private key file for HTTPS.
		// Required if you use your own certificates
		CertificateKey string `yaml:"certificate_key"`

		// CertificatePem is the path or content of the TLS certificate file for HTTPS.
		// Required if you use your own certificates
		CertificatePem string `yaml:"certificate_pem"`

		// ServerName is the short name identifier for the server.
		ServerName string `yaml:"server_name"`

		// ServerFQDN is the fully qualified domain name of the server.
		ServerFQDN string `yaml:"server_fqdn"`

		// DockerRegistry specifies the default container registry URL for pulling images.
		DockerRegistry string `yaml:"docker_registry"`

		// DockerAuthentication holds the authentication token or credentials for the default Docker registry.
		DockerAuthentication string `yaml:"docker_authentication"`

		// DockerCredentials contains multiple registry→secret pairs for authenticating to container registries.
		// Used by clients to access private registries.
		DockerCredentials []DockerCredential `yaml:"docker_credentials"`

		// SwapProportion defines the fraction of /scratch dedicated to swap on worker automated deploy.
		// Default 0.10. A recruiter can override this per step via its own swap_proportion
		// (0 disables swap entirely on that recruiter's workers; see Recruiter.swap_proportion).
		SwapProportion float32 `yaml:"swap_proportion" default:"0.1"`

		// WorkerToken is a secret token used to authenticate worker nodes.
		WorkerToken string `yaml:"worker_token"`

		// JwtSecret is the secret key used to sign JWT tokens.
		JwtSecret string `yaml:"jwt_secret"`

		// RecruitmentInterval sets the interval in seconds for recruiting new workers.
		RecruitmentInterval int `yaml:"recruiter_interval" default:"5"`

		// IdleTimeout defines the timeout in seconds after which idle workers are considered for shutdown.
		IdleTimeout int `yaml:"idle_timeout" default:"300"`

		// NewWorkerIdleTimeout is the timeout in seconds for newly started workers before they are considered idle.
		NewWorkerIdleTimeout int `yaml:"new_worker_idle_timeout" default:"900"`

		// OfflineTimeout is the number of seconds without a worker ping
		// after which the server flips the worker's status to O (offline)
		// AND reclaims its assigned/accepted/downloading tasks (resets
		// them to P so other workers can pick them up). 10 minutes is the
		// default: long enough to tolerate normal network hiccups,
		// short-lived upgrades, and worker restarts, short enough that a
		// truly dead worker doesn't strand its share of the queue for
		// hours. Set lower (e.g. 60) for latency-sensitive fleets where
		// task reclaim should be aggressive; set higher (e.g. 1800) for
		// flaky networks where transient ping loss is common.
		OfflineTimeout int `yaml:"offline_timeout" default:"600"`

		// WorkerStatsRetentionHours controls how long historical per-worker
		// stats samples (worker_stats_history table) are kept. Each ping
		// writes one row with the worker's current and peak-since-last-ping
		// cpu/mem/iowait, so a completed workflow can be inspected days
		// later ("what did this workflow actually need?"). 0 disables the
		// feature entirely — no rows are written and the sweep goroutine
		// does not run. Default 168 (7 days) comfortably covers a Friday
		// workflow inspected on Monday.
		WorkerStatsRetentionHours int `yaml:"worker_stats_retention_hours" default:"168"`

		// TaskDownloadTimeout is the timeout in seconds for task data downloads.
		TaskDownloadTimeout int `yaml:"task_download_timeout" default:"600"`

		// TaskExecutionTimeout is the timeout in seconds for task execution.
		// A value of 0 disables the timeout.
		TaskExecutionTimeout int `yaml:"task_execution_timeout" default:"0"`

		// TaskUploadTimeout is the timeout in seconds for uploading task results.
		TaskUploadTimeout int `yaml:"task_upload_timeout" default:"600"`

		// ConsideredLostTimeout is the timeout in seconds after which a task is considered lost.
		ConsideredLostTimeout int `yaml:"considered_lost_timeout" default:"300" `

		// AdminUser is the username for the administrator account.
		AdminUser string `yaml:"admin_user" default:"admin"`

		// AdminHashedPassword is the hashed password for the administrator account.
		// It can be generated by CLI : `scitq hashpassword MySuperPassword`
		AdminHashedPassword string `yaml:"admin_hashed_password" default:""`

		// AdminEmail is the email address of the administrator.
		AdminEmail string `yaml:"admin_email" default:""`

		// DisableHTTPS disables HTTPS support when set to true.
		DisableHTTPS bool `yaml:"disable_https" default:"false"`

		// DisableGRPCWeb disables gRPC-Web support when set to true.
		// Used for test only
		DisableGRPCWeb bool `yaml:"disable_grpcweb" default:"false"`

		// HTTPSPort is the TCP port used for HTTPS connections.
		HTTPSPort int `yaml:"https_port" default:"443"`

		// HTTPPort is the TCP port used for plain HTTP (MCP, API endpoints,
		// the worker-binary download endpoint). Only consulted when HTTPS
		// is disabled (e.g. integration tests). Default 0 means "derive
		// from Port + 1", which is what production deployments rely on.
		// The override exists for tests that need to reserve two
		// independent free ports (without the +1 assumption that
		// otherwise races with parallel tests reserving consecutive
		// numbers).
		HTTPPort int `yaml:"http_port" default:"0"`

		// GRPCDSLTimeout is the timeout (in seconds) for DSL scripts which can take a long time to complete.
		GRPCDSLTimeout int `yaml:"grpc_dsl_timeout" default:"1800"`

		// WorkerRetention is the number of days to retain soft-deleted workers before pruning.
		WorkerRetention int `yaml:"worker_retention" default:"30"`

		// ModulesRoot is the directory where YAML module files are stored as
		// the canonical source of truth. Each module row in the `module`
		// table points at `{ModulesRoot}/<path>/<version>.yaml`. Admins can
		// inspect / grep / rsync / git-track this directory; losing the DB
		// is recoverable by reindexing the tree at startup. See
		// specs/module_library.md.
		ModulesRoot string `yaml:"modules_root" default:"/var/lib/scitq/modules"`

		// AutoupgradeModules, when true (default), triggers the server to run
		// `module upgrade --apply` semantics at every startup so bundled YAML
		// modules shipped in the scitq2_modules Python package appear in the
		// server-side module library without an explicit admin step. Set to
		// false for frozen-library environments (audit-sensitive sites, CI)
		// where every module change must be gated through a manual review.
		// See specs/module_library.md.
		AutoupgradeModules bool `yaml:"autoupgrade_modules" default:"true"`

		// AutoupgradeExclude is a list of glob-style path patterns that the
		// auto-upgrade step will skip. `*` matches a single path segment,
		// `**` matches any number of segments; everything else is literal.
		// Examples:
		//   - "metagenomics/*"     # skip every direct child of the namespace
		//   - "genomics/multiqc"   # skip exactly this module
		//   - "internal/**"        # skip every module under internal/ recursively
		// Excluded modules are NOT reinstated on subsequent startups; does
		// not retroactively delete rows already in the library (use
		// `scitq module delete` for that, then add to this list to prevent
		// reinstatement).
		AutoupgradeExclude []string `yaml:"autoupgrade_exclude"`
	} `yaml:"scitq"`

	// Providers contains configurations for different cloud providers supported by scitq.
	// Each provider can use multiple account, so you can have several config called Primary, Secondary etc.
	// For OVH, use an Openstack account (that you can name OVH)
	// see the example for details
	Providers struct {
		Azure     map[string]*AzureConfig        `yaml:"azure"`
		Openstack map[string]*OpenstackConfig    `yaml:"openstack"`
		Fake      map[string]*FakeProviderConfig `yaml:"fake"`
		Local     map[string]*LocalConfig        `yaml:"local"`
	} `yaml:"providers"`

	// Rclone holds configuration mappings for rclone integrations.
	// Create your config using native rclone with `rclone config`
	// then export the config to `scitq.yaml` with the CLI `scitq config import-rclone >> /etc/scitq.yaml`
	Rclone map[string]map[string]string `yaml:"rclone"`

	// Notifications configures user-facing convenience notifications
	// (workflow completion, ...). NOT the admin monitoring path —
	// leaked workers etc. surface via server/metrics for Zabbix /
	// Prometheus / Grafana consumption. See server/notifications.
	Notifications NotificationsConfig `yaml:"notifications"`
}

type DockerCredential struct {
	Registry string `yaml:"registry"`
	Secret   string `yaml:"secret"`
}

type FakeProviderConfig struct {
	Name          string           `yaml:"-"`
	DefaultRegion string           `yaml:"default_region"`
	Regions       []string         `yaml:"regions"`
	Quotas        map[string]Quota `yaml:"quotas"`

	// The following fields are used only in testing contexts to allow
	// the fake provider to automatically launch worker clients against
	// a running test server.
	AutoLaunch bool   `yaml:"auto_launch"`
	ServerAddr string `yaml:"server_addr"`
	DeployTime int    `yaml:"delayed_deploy" default:"0"` // seconds to wait before launching workers
}

type LocalConfig struct {
	Name                string            `yaml:"-"`
	DefaultRegion       string            `yaml:"default_region"`
	Regions             []string          `yaml:"regions"`
	LocalWorkspaceRoots map[string]string `yaml:"local_workspaces"` // key: region (or "*" wildcard); value: workspace URI for that region
	LocalResourceRoots  map[string]string `yaml:"local_resources"`  // key: region (or "*" wildcard); value: resource-root URI for that region
}

type NotificationChannel struct {
	// Name identifies this channel in the routes table and in log
	// lines. Must be non-empty.
	Name string `yaml:"name"`

	// Kind picks the backend: "zulip" | "webhook" | "log".
	Kind string `yaml:"kind"`

	// URL is the endpoint the backend POSTs to. Required for zulip
	// and webhook; ignored by log.
	URL string `yaml:"url"`

	// Headers are added to every HTTP request the backend makes.
	// Useful for Authorization / X-Auth-Token style tokens on
	// generic webhooks. Ignored by zulip (its URL already carries
	// the api_key as a query param).
	Headers map[string]string `yaml:"headers"`

	// Options carries backend-specific string knobs — kept as a
	// generic map so a new backend can pick up new keys without
	// widening this struct. Known keys:
	//   zulip:   "topic" (override the URL's default topic)
	//   webhook: "template" (Go text/template body),
	//            "method" (default POST),
	//            "content_type" (default application/json)
	Options map[string]string `yaml:"options"`
}

type NotificationRoute struct {
	// Event is the routing key. Currently: "workflow.terminal".
	Event string `yaml:"event"`

	// Channels lists channel names (matching NotificationChannel.Name)
	// that receive this event.
	Channels []string `yaml:"channels"`
}

type NotificationsConfig struct {
	// AlwaysLog writes every dispatched notification to the server log
	// in addition to routing to configured channels. Useful as an
	// audit trail or to smoke-test rules without wiring a real backend.
	AlwaysLog bool `yaml:"always_log" default:"false"`

	// Channels declares the delivery targets. See NotificationChannel
	// for the shape.
	Channels []NotificationChannel `yaml:"channels"`

	// Routes maps events to channel names. One event can fan out to
	// several channels; one channel can subscribe to several events.
	Routes []NotificationRoute `yaml:"routes"`
}

type OpenstackConfig struct {
	Name       string `yaml:"-"`
	AuthURL    string `yaml:"auth_url"`
	Username   string `yaml:"username"`
	Password   string `yaml:"password"`
	DomainName string `yaml:"domain_name"`
	DomainID   string `yaml:"domain_id"`
	TenantName string `yaml:"tenant_name"`

	// Keystone project identifiers (either one can be used)
	ProjectID   string `yaml:"project_id"`
	ProjectName string `yaml:"project_name"`

	// Domain scoping (Keystone v3)
	UserDomainName  string `yaml:"user_domain_name"`
	ProjectDomainID string `yaml:"project_domain_id"`

	// Optional: prefer Application Credentials when provided (portable OpenStack)
	ApplicationCredentialID     string `yaml:"application_credential_id"`
	ApplicationCredentialSecret string `yaml:"application_credential_secret"`

	// Optional interface selection for service endpoints (public/internal/admin)
	Interface string `yaml:"interface"` // Unsure this is used - prefer NetworkID

	// Optional: Keystone identity API version (default 3)
	IdentityAPIVersion int `yaml:"identity_api_version" default:"3"`

	DefaultRegion string `yaml:"region"`
	ImageID       string `yaml:"image_id"`
	// Image (name or UUID) booted for workers recruited on a
	// has_gpu=true flavor. Default: OVH's "NVIDIA GPU Cloud (NGC)"
	// — Ubuntu 22.04 with NVIDIA drivers + nvidia-container-toolkit
	// + docker preconfigured for `--gpus all`. Override per
	// environment if your OpenStack catalog uses a different name;
	// `openstack image list --tag gpu` lists candidates.
	GPUImageID          string                 `yaml:"gpu_image_id" default:"NVIDIA GPU Cloud (NGC)"`
	FlavorID            string                 `yaml:"flavor_id"`
	NetworkID           string                 `yaml:"network_id"`
	ExtNetworkID        string                 `yaml:"ext_network_id"`
	Quotas              map[string]Quota       `yaml:"quotas"` // key: region
	Regions             []string               `yaml:"regions"`
	Custom              map[string]interface{} `yaml:"custom"`             // Vendor-specific custom settings
	UpdatePeriodicity   string                 `yaml:"update_periodicity"` // Update periodicity in minutes
	LocalWorkspaceRoots map[string]string      `yaml:"local_workspaces"`   // key: region (or "*" wildcard); value: workspace URI for that region
	LocalResourceRoots  map[string]string      `yaml:"local_resources"`    // key: region (or "*" wildcard); value: resource-root URI for that region
	Keypair             string                 `yaml:"keypair"`            // Name of the keypair to use for SSH access
	// See AzureConfig.FlavorIncludePatterns / FlavorExcludePatterns —
	// same semantics for OVH/Openstack catalogs.
	FlavorIncludePatterns []string `yaml:"flavor_include_patterns"`
	FlavorExcludePatterns []string `yaml:"flavor_exclude_patterns"`
}

type ProviderConfig interface {
	GetRegions() []string
	SetRegions([]string)
	GetQuotas() map[string]Quota
	GetUpdatePeriodicity() time.Duration
	GetName() string
	SetName(string)
	GetDefaultRegion() string
	GetWorkspaceRoot(region string) (string, bool)
	GetResourceRoot(region string) (string, bool)
}

type Quota struct {
	MaxCPU       int32   `yaml:"cpu"`
	MaxMemGB     float32 `yaml:"mem,omitempty"`       // optional
	MaxInstances int32   `yaml:"instances,omitempty"` // optional, 0 = unlimited (can also be learned from failures)
}