tangled.org
Monorepo for Tangled
tangled.org
ptr.pet
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}: let
7 cfg = config.services.tangled.spindle;
8in
9 with lib; {
10 options = let
11 processOptions = {
12 package = mkOption {
13 type = types.package;
14 description = "Package to use for the spindle";
15 };
16
17 server = {
18 listenAddr = mkOption {
19 type = types.str;
20 default = "0.0.0.0:6555";
21 description = "Address to listen on";
22 };
23
24 dbPath = mkOption {
25 type = types.path;
26 default = "/var/lib/spindle/spindle.db";
27 description = "Path to the database file";
28 };
29
30 repoDir = mkOption {
31 type = types.path;
32 default = "/var/lib/spindle/repos";
33 description = "Path where synced git repositories live";
34 };
35
36 hostname = mkOption {
37 type = types.str;
38 example = "my.spindle.com";
39 description = "Hostname for the server (required)";
40 };
41
42 plcUrl = mkOption {
43 type = types.str;
44 default = "https://plc.directory";
45 description = "atproto PLC directory";
46 };
47
48 jetstreamEndpoint = mkOption {
49 type = types.str;
50 default = "wss://jetstream1.us-west.bsky.network/subscribe";
51 description = "Jetstream endpoint to subscribe to";
52 };
53
54 dev = mkOption {
55 type = types.bool;
56 default = false;
57 description = "Enable development mode (disables signature verification)";
58 };
59
60 owner = mkOption {
61 type = types.str;
62 example = "did:plc:qfpnj4og54vl56wngdriaxug";
63 description = "DID of owner (required)";
64 };
65
66 maxJobCount = mkOption {
67 type = types.int;
68 default = 2;
69 example = 5;
70 description = "Maximum number of concurrent jobs to run";
71 };
72
73 queueSize = mkOption {
74 type = types.int;
75 default = 100;
76 example = 100;
77 description = "Maximum number of jobs queue up";
78 };
79
80 secrets = {
81 provider = mkOption {
82 type = types.str;
83 default = "sqlite";
84 description = "Backend to use for secret management, valid options are 'sqlite', and 'openbao'.";
85 };
86
87 openbao = {
88 proxyAddr = mkOption {
89 type = types.str;
90 default = "http://127.0.0.1:8200";
91 description = "Address of the OpenBAO proxy server";
92 };
93 mount = mkOption {
94 type = types.str;
95 default = "spindle";
96 description = "Mount path in OpenBAO to read secrets from";
97 };
98 };
99 };
100
101 tap = {
102 embed = mkOption {
103 type = types.bool;
104 default = true;
105 description = "Run an embedded tap inside the spindle process";
106 };
107
108 url = mkOption {
109 type = types.str;
110 default = "http://[::1]:2480";
111 description = "URL the spindle's tap client dials";
112 };
113
114 bind = mkOption {
115 type = types.str;
116 default = "[::1]:2480";
117 description = "Loopback address the embedded tap server listens on";
118 };
119
120 dbPath = mkOption {
121 type = types.path;
122 default = "/var/lib/spindle/tap.db";
123 description = "Path to the embedded tap sqlite database";
124 };
125
126 relayUrl = mkOption {
127 type = types.str;
128 default = "https://bsky.network";
129 description = "Relay used by the embedded tap firehose";
130 };
131 };
132 };
133
134 artifactStores = {
135 disk.dir = mkOption {
136 type = types.path;
137 default = "/var/log/spindle";
138 description = "Root directory for disk artifacts";
139 };
140
141 s3.bucket = mkOption {
142 type = types.str;
143 default = "tangled-logs";
144 description = "S3 bucket for artifacts";
145 };
146
147 s3.region = mkOption {
148 type = types.str;
149 default = "us-east-1";
150 description = "AWS region for the artifact bucket";
151 };
152 };
153
154 pipelines = {
155 workflowTimeout = mkOption {
156 type = types.str;
157 default = "5m";
158 description = "Timeout for a whole workflow, covering the wait for a concurrency slot, setup, and every step in it";
159 };
160
161 nixery = {
162 nixery = mkOption {
163 type = types.str;
164 default = "nixery.tangled.sh"; # note: this is *not* on tangled.org yet
165 description = "Nixery instance to use";
166 };
167
168 maxJobMemoryMb = mkOption {
169 type = types.int;
170 default = 6144;
171 description = "Memory limit per nixery workflow container in MiB (default 6 GiB)";
172 };
173 maxConcurrentWorkflows = mkOption {
174 type = types.int;
175 default = 8;
176 description = "Maximum number of nixery workflows running simultaneously. Zero disables this limit.";
177 };
178 };
179
180 microvm = {
181 enableKVM = mkOption {
182 type = types.bool;
183 default = true;
184 description = "Enable KVM hardware acceleration";
185 };
186
187 imageDir = mkOption {
188 type = types.str;
189 default = "/var/lib/spindle/images";
190 description = "Directory containing microVM image spec JSONs or image spec directories";
191 };
192 overlayDir = mkOption {
193 type = types.str;
194 default = "/tmp";
195 description = "Directory to store microVM temporary overlay files";
196 };
197 defaultImage = mkOption {
198 type = types.str;
199 default = "nixos";
200 description = "Default microVM image spec to use if none is specified in workflow";
201 };
202 agentPort = mkOption {
203 type = types.port;
204 default = 10240;
205 description = "Host vsock port the microVM agent connects back to";
206 };
207
208 limits = {
209 total = {
210 memoryMiB = mkOption {
211 type = types.int;
212 default = 0;
213 description = "Maximum declared guest memory in MiB allowed across all running microVM workflows. Zero disables this limit.";
214 };
215 vcpus = mkOption {
216 type = types.int;
217 default = 0;
218 description = "Maximum declared vCPUs allowed across all running microVM workflows. Zero disables this limit.";
219 };
220 diskMiB = mkOption {
221 type = types.int;
222 default = 0;
223 description = "Maximum declared disk in MiB allowed across all running microVM workflows. Zero disables this limit.";
224 };
225 };
226
227 workflow = {
228 memoryMiB = mkOption {
229 type = types.int;
230 default = 0;
231 description = "Maximum declared guest memory in MiB allowed for a single microVM workflow. Zero disables this limit.";
232 };
233 vcpus = mkOption {
234 type = types.int;
235 default = 0;
236 description = "Maximum declared vCPUs allowed for a single microVM workflow. Zero disables this limit.";
237 };
238 diskMiB = mkOption {
239 type = types.int;
240 default = 0;
241 description = "Maximum declared disk in MiB allowed for a single microVM workflow. Zero disables this limit.";
242 };
243 };
244 };
245
246 cgroup = {
247 enable = mkOption {
248 type = types.bool;
249 default = false;
250 description = "Enable cgroup v2 containment for microVM processes.";
251 };
252 parent = mkOption {
253 type = types.str;
254 default = "self";
255 description = "Parent cgroup for microVM workflow cgroups. Use 'self' to resolve the spindle service cgroup.";
256 };
257 pidsMax = mkOption {
258 type = types.int;
259 default = 4096;
260 description = "Maximum number of processes allowed in each microVM workflow cgroup.";
261 };
262 swapMaxMiB = mkOption {
263 type = types.int;
264 default = 0;
265 description = "Maximum swap in MiB allowed in each microVM workflow cgroup. Zero disables swap.";
266 };
267 supervisorMinMiB = mkOption {
268 type = types.int;
269 default = 512;
270 description = ''
271 Amount of memory in MiB that will be protected by the cgroup for the spindle
272 (allowing it to not get OOMed first.)
273 '';
274 };
275 };
276
277 debugSsh = {
278 enable = mkOption {
279 type = types.bool;
280 default = false;
281 description = ''
282 Enable the debug ssh server that lets authorized users ssh into a
283 failed microVM to debug it.
284 '';
285 };
286 listenAddr = mkOption {
287 type = types.str;
288 default = "0.0.0.0:2222";
289 example = "0.0.0.0:2225";
290 description = "Address for the debug ssh server to listen on.";
291 };
292 host = mkOption {
293 type = types.str;
294 default = "";
295 example = "127.0.0.1";
296 description = "Host reached from the SSH jump host.";
297 };
298 jumpHost = mkOption {
299 type = types.str;
300 default = "";
301 example = "spindle.example.com";
302 description = "SSH jump host used in the printed debug command.";
303 };
304 hostKeyPath = mkOption {
305 type = with types; nullOr path;
306 default = null;
307 example = "/var/lib/spindle/debug_ssh_host_key";
308 description = ''
309 Path to the ssh host key for the debug server. If null, one is generated
310 once and persisted next to the spindle db.
311 '';
312 };
313 gracePeriod = mkOption {
314 type = types.str;
315 default = "5m";
316 description = ''
317 How long a failed workflow's microVM is kept alive for the user to ssh in.
318 '';
319 };
320 };
321 };
322
323 nixCache = {
324 readUrls = mkOption {
325 type = types.listOf types.str;
326 default = [];
327 example = ["http://ncps.internal:8501" "ssh-ng://user@my-awesome-cache"];
328 description = "Nix binary cache URLs the Spindle guest should read from.";
329 };
330
331 trustedPublicKeys = mkOption {
332 type = types.listOf types.str;
333 default = [];
334 example = ["internal-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="];
335 description = "Public keys trusted for the configured Nix binary caches.";
336 };
337
338 uploadUrl = mkOption {
339 type = types.str;
340 default = "";
341 example = "local";
342 description = "Optional cache upload URL used by live cache import paths.";
343 };
344 };
345 };
346
347 environmentFile = mkOption {
348 type = with types; nullOr path;
349 default = null;
350 example = "/etc/spindle.env";
351 description = ''
352 Additional environment file as defined in {manpage}`systemd.exec(5)`.
353
354 Sensitive secrets such as {env}`AWS_SECRET_ACCESS_KEY`,
355 {env}`AWS_ACCESS_KEY_ID`, {env}`AWS_REGION`
356 may be passed to the service
357 without making them world readable in the nix store.
358 '';
359 };
360 };
361
362 localServiceType = name:
363 types.submodule ({config, ...}: {
364 options =
365 processOptions
366 // {
367 generateToken = mkOption {
368 type = types.bool;
369 default = false;
370 description = "Generate the token file once at a persistent path.";
371 };
372
373 millUrl = mkOption {
374 type = types.str;
375 default = "ws://127.0.0.1:6555/mill";
376 description = "Mill URL used by this executor.";
377 };
378
379 seats = mkOption {
380 type = types.ints.positive;
381 default = 4;
382 description = "Jobs this executor may run at once.";
383 };
384
385 stateDirectory = mkOption {
386 type = types.str;
387 default = "spindle-executor-${name}";
388 description = "State directory used by this executor.";
389 };
390 };
391
392 config = mkMerge [
393 {
394 package = mkDefault cfg.package;
395 server = mkDefault cfg.server;
396 artifactStores = mkDefault cfg.artifactStores;
397 pipelines = mkDefault cfg.pipelines;
398 environmentFile = mkDefault cfg.environmentFile;
399 }
400 {
401 server.listenAddr = mkOverride 900 "127.0.0.1:0";
402 server.dbPath = mkOverride 900 "/var/lib/${config.stateDirectory}/spindle.db";
403 server.repoDir = mkOverride 900 "/var/lib/${config.stateDirectory}/repos";
404 server.hostname = mkOverride 900 name;
405 server.owner = mkOverride 900 "";
406 pipelines.microvm.debugSsh.listenAddr = mkOverride 900 "127.0.0.1:${toString cfg.mill.debugExecutorPort}";
407 }
408 ];
409 });
410
411 executorType = types.submodule ({name, ...}: {
412 options = {
413 tokenFile = mkOption {
414 type = types.str;
415 description = "File containing this executor's token.";
416 };
417
418 labels = mkOption {
419 type = types.listOf types.str;
420 default = [];
421 description = "Labels this executor may use.";
422 };
423
424 localService = mkOption {
425 type = with types; nullOr (localServiceType name);
426 default = null;
427 description = "Service running this executor on the mill host.";
428 };
429 };
430 });
431
432 dedicatedExecutorType = types.submodule {
433 options = {
434 name = mkOption {
435 type = types.str;
436 description = "Name registered for this executor.";
437 };
438
439 millUrl = mkOption {
440 type = types.str;
441 description = "Mill URL used by this executor.";
442 };
443
444 tokenFile = mkOption {
445 type = types.str;
446 description = "File containing this executor's token.";
447 };
448
449 labels = mkOption {
450 type = types.listOf types.str;
451 default = [];
452 description = "Labels this executor may use.";
453 };
454
455 seats = mkOption {
456 type = types.ints.positive;
457 default = 4;
458 description = "Jobs this executor may run at once.";
459 };
460 };
461 };
462 in {
463 services.tangled.spindle =
464 processOptions
465 // {
466 enable = mkOption {
467 type = types.bool;
468 default = false;
469 description = "Enable a tangled spindle";
470 };
471
472 role = mkOption {
473 type = types.enum ["standalone" "mill" "executor"];
474 default = "standalone";
475 description = "How the main spindle runs.";
476 };
477
478 executor = mkOption {
479 type = with types; nullOr dedicatedExecutorType;
480 default = null;
481 description = "Settings used when the main spindle runs as an executor.";
482 };
483
484 mill = {
485 artifactStore = mkOption {
486 type = types.enum ["disk" "s3"];
487 default = "s3";
488 description = "Artifact store shared by the mill and its executors.";
489 };
490
491 drainTimeout = mkOption {
492 type = types.ints.positive;
493 default = 1200;
494 description = "Seconds an executor waits for running jobs to finish before stopping.";
495 };
496
497 executors = mkOption {
498 type = types.attrsOf executorType;
499 default = {};
500 description = "Executors allowed to connect to this mill.";
501 };
502
503 jumpListenAddr = mkOption {
504 type = types.str;
505 default = "";
506 example = "0.0.0.0:22";
507 description = "Address for the mill's restricted debug SSH jump server.";
508 };
509
510 jumpHostKeyPath = mkOption {
511 type = with types; nullOr path;
512 default = null;
513 example = "/var/lib/spindle/debug_jump_host_key";
514 description = "Path to the debug SSH jump server host key.";
515 };
516
517 debugExecutorPort = mkOption {
518 type = types.port;
519 default = 2223;
520 description = "Private debug SSH port shared by executors.";
521 };
522
523 maxJumpConnections = mkOption {
524 type = types.ints.positive;
525 default = 128;
526 description = "Maximum concurrent connections to the mill's debug SSH jump server.";
527 };
528 };
529 };
530 };
531 config = let
532 deps = [
533 pkgs.git
534 pkgs.qemu
535 pkgs.e2fsprogs
536 pkgs.slirp4netns
537 pkgs.iproute2
538 pkgs.util-linux
539 config.nix.package
540 ];
541 localExecutors = filterAttrs (_: executor: executor.localService != null) cfg.mill.executors;
542 localServices = mapAttrs (_: executor: executor.localService) localExecutors;
543 localAgentPorts = mapAttrsToList (_: service: service.pipelines.microvm.agentPort) localServices;
544 localStateDirectories = mapAttrsToList (_: service: service.stateDirectory) localServices;
545 localDbPaths = mapAttrsToList (_: service: toString service.server.dbPath) localServices;
546 localRepoDirs = mapAttrsToList (_: service: toString service.server.repoDir) localServices;
547 debugServices = filterAttrs (_: service: service.pipelines.microvm.debugSsh.enable) localServices;
548 localDebugSshListenAddrs =
549 mapAttrsToList (_: service: service.pipelines.microvm.debugSsh.listenAddr) debugServices;
550 hasRegistrations = cfg.role == "mill" && cfg.mill.executors != {};
551 isDedicatedExecutor = cfg.role == "executor" && cfg.executor != null;
552 executorServiceName = name: "spindle-executor-${name}";
553 executorExecStart = name: package:
554 pkgs.writeShellScript name ''
555 export SPINDLE_MILL_SHARED_SECRET="$(${pkgs.coreutils}/bin/cat "$CREDENTIALS_DIRECTORY/mill-token")"
556 exec ${pkgs.coreutils}/bin/env \
557 SPINDLE_ROLE=executor \
558 SPINDLE_MILL_ARTIFACT_STORE=${cfg.mill.artifactStore} \
559 SPINDLE_MILL_DRAIN_TIMEOUT=${toString cfg.mill.drainTimeout}s \
560 SPINDLE_MILL_JUMP_LISTEN_ADDR= \
561 SPINDLE_MILL_JUMP_HOST_KEY_PATH= \
562 ${package}/bin/spindle
563 '';
564
565 processEnvironment = instance: [
566 "SPINDLE_SERVER_LISTEN_ADDR=${instance.server.listenAddr}"
567 "SPINDLE_SERVER_DB_PATH=${instance.server.dbPath}"
568 "SPINDLE_SERVER_REPO_DIR=${instance.server.repoDir}"
569 "SPINDLE_SERVER_HOSTNAME=${instance.server.hostname}"
570 "SPINDLE_SERVER_PLC_URL=${instance.server.plcUrl}"
571 "SPINDLE_SERVER_JETSTREAM_ENDPOINT=${instance.server.jetstreamEndpoint}"
572 "SPINDLE_SERVER_DEV=${lib.boolToString instance.server.dev}"
573 "SPINDLE_SERVER_OWNER=${instance.server.owner}"
574 "SPINDLE_SERVER_MAX_JOB_COUNT=${toString instance.server.maxJobCount}"
575 "SPINDLE_SERVER_QUEUE_SIZE=${toString instance.server.queueSize}"
576 "SPINDLE_SERVER_SECRETS_PROVIDER=${instance.server.secrets.provider}"
577 "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${instance.server.secrets.openbao.proxyAddr}"
578 "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${instance.server.secrets.openbao.mount}"
579 "SPINDLE_SERVER_TAP_EMBED=${lib.boolToString instance.server.tap.embed}"
580 "SPINDLE_SERVER_TAP_URL=${instance.server.tap.url}"
581 "SPINDLE_SERVER_TAP_BIND=${instance.server.tap.bind}"
582 "SPINDLE_SERVER_TAP_DB_PATH=${instance.server.tap.dbPath}"
583 "SPINDLE_SERVER_TAP_RELAY_URL=${instance.server.tap.relayUrl}"
584 "SPINDLE_NIXERY_PIPELINES_NIXERY=${instance.pipelines.nixery.nixery}"
585 "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${instance.pipelines.workflowTimeout}"
586 "SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB=${toString instance.pipelines.nixery.maxJobMemoryMb}"
587 "SPINDLE_NIXERY_PIPELINES_MAX_CONCURRENT_WORKFLOWS=${toString instance.pipelines.nixery.maxConcurrentWorkflows}"
588 "SPINDLE_MICROVM_PIPELINES_IMAGE_DIR=${instance.pipelines.microvm.imageDir}"
589 "SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR=${instance.pipelines.microvm.overlayDir}"
590 "SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE=${instance.pipelines.microvm.defaultImage}"
591 "SPINDLE_MICROVM_PIPELINES_AGENT_PORT=${toString instance.pipelines.microvm.agentPort}"
592 "SPINDLE_MICROVM_PIPELINES_ENABLE_KVM=${lib.boolToString instance.pipelines.microvm.enableKVM}"
593 "SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT=${instance.pipelines.workflowTimeout}"
594 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB=${toString instance.pipelines.microvm.limits.total.memoryMiB}"
595 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS=${toString instance.pipelines.microvm.limits.total.vcpus}"
596 "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB=${toString instance.pipelines.microvm.limits.total.diskMiB}"
597 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_MEMORY_MIB=${toString instance.pipelines.microvm.limits.workflow.memoryMiB}"
598 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_VCPUS=${toString instance.pipelines.microvm.limits.workflow.vcpus}"
599 "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_DISK_MIB=${toString instance.pipelines.microvm.limits.workflow.diskMiB}"
600 "SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS=${lib.boolToString instance.pipelines.microvm.cgroup.enable}"
601 "SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT=${instance.pipelines.microvm.cgroup.parent}"
602 "SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX=${toString instance.pipelines.microvm.cgroup.pidsMax}"
603 "SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB=${toString instance.pipelines.microvm.cgroup.swapMaxMiB}"
604 "SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB=${toString instance.pipelines.microvm.cgroup.supervisorMinMiB}"
605 "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_ENABLED=${lib.boolToString instance.pipelines.microvm.debugSsh.enable}"
606 "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_LISTEN_ADDR=${instance.pipelines.microvm.debugSsh.listenAddr}"
607 "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST=${instance.pipelines.microvm.debugSsh.host}"
608 "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_JUMP_HOST=${instance.pipelines.microvm.debugSsh.jumpHost}"
609 "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST_KEY_PATH=${optionalString (instance.pipelines.microvm.debugSsh.hostKeyPath != null) (toString instance.pipelines.microvm.debugSsh.hostKeyPath)}"
610 "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_GRACE_PERIOD=${instance.pipelines.microvm.debugSsh.gracePeriod}"
611 "SPINDLE_NIX_CACHE_READ_URLS=${concatStringsSep "," instance.pipelines.nixCache.readUrls}"
612 "SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=${concatStringsSep "," instance.pipelines.nixCache.trustedPublicKeys}"
613 "SPINDLE_NIX_CACHE_UPLOAD_URL=${instance.pipelines.nixCache.uploadUrl}"
614 "SPINDLE_ARTIFACT_STORES_DISK_DIR=${instance.artifactStores.disk.dir}"
615 "SPINDLE_ARTIFACT_STORES_S3_BUCKET=${instance.artifactStores.s3.bucket}"
616 "SPINDLE_ARTIFACT_STORES_S3_REGION=${instance.artifactStores.s3.region}"
617 "SPINDLE_MILL_ARTIFACT_STORE=${cfg.mill.artifactStore}"
618 "SPINDLE_MILL_DRAIN_TIMEOUT=${toString cfg.mill.drainTimeout}s"
619 ];
620
621 connectionEnvironment = executor: [
622 "SPINDLE_MILL_URL=${executor.millUrl}"
623 "SPINDLE_MILL_SEATS=${toString executor.seats}"
624 "SPINDLE_MILL_LABELS=${concatStringsSep "," executor.labels}"
625 ];
626
627 mainEnvironment =
628 processEnvironment cfg
629 ++ [
630 "SPINDLE_ROLE=${cfg.role}"
631 "SPINDLE_MILL_JUMP_LISTEN_ADDR=${cfg.mill.jumpListenAddr}"
632 "SPINDLE_MILL_JUMP_HOST_KEY_PATH=${optionalString (cfg.mill.jumpHostKeyPath != null) (toString cfg.mill.jumpHostKeyPath)}"
633 "SPINDLE_MILL_DEBUG_EXECUTOR_PORT=${toString cfg.mill.debugExecutorPort}"
634 "SPINDLE_MILL_MAX_JUMP_CONNECTIONS=${toString cfg.mill.maxJumpConnections}"
635 ]
636 ++ optionals isDedicatedExecutor (connectionEnvironment cfg.executor);
637
638 localExecutorEnvironment = executor:
639 processEnvironment executor.localService
640 ++ [
641 "SPINDLE_ROLE=executor"
642 "SPINDLE_SERVER_LOG_DIR=/var/log/${executor.localService.stateDirectory}"
643 ]
644 ++ connectionEnvironment {
645 inherit (executor) labels;
646 inherit (executor.localService) millUrl seats;
647 };
648
649 registrationCommands = concatStringsSep "\n" (mapAttrsToList (
650 name: executor: let
651 tokenFile = escapeShellArg executor.tokenFile;
652 tokenDir = escapeShellArg (builtins.dirOf executor.tokenFile);
653 tokenTemplate = escapeShellArg "${executor.tokenFile}.tmp.XXXXXX";
654 generateCommand = escapeShellArgs [
655 "${cfg.package}/bin/spindle"
656 "mill"
657 "executor"
658 "token"
659 "generate"
660 "--db"
661 (toString cfg.server.dbPath)
662 ];
663 registerCommand = escapeShellArgs (
664 [
665 "${cfg.package}/bin/spindle"
666 "mill"
667 "executor"
668 "add"
669 "--db"
670 (toString cfg.server.dbPath)
671 "--token-file"
672 executor.tokenFile
673 ]
674 ++ concatMap (label: ["--label" label]) executor.labels
675 ++ [name]
676 );
677 emptyGeneratedToken = escapeShellArg "generated token for ${name} is empty";
678 missingToken = escapeShellArg "token file for ${name} is missing or empty: ${executor.tokenFile}";
679 in ''
680 ${optionalString (executor.localService != null && executor.localService.generateToken) ''
681 if [ ! -e ${tokenFile} ]; then
682 if [ ! -d ${tokenDir} ]; then
683 ${pkgs.coreutils}/bin/install -d -m 0700 ${tokenDir}
684 fi
685 umask 0077
686 tmp="$(${pkgs.coreutils}/bin/mktemp ${tokenTemplate})"
687 trap '${pkgs.coreutils}/bin/rm -f "$tmp"' EXIT
688 ${generateCommand} > "$tmp"
689 if [ ! -s "$tmp" ]; then
690 printf '%s\n' ${emptyGeneratedToken} >&2
691 exit 1
692 fi
693 ${pkgs.coreutils}/bin/chmod 0600 "$tmp"
694 ${pkgs.coreutils}/bin/mv "$tmp" ${tokenFile}
695 trap - EXIT
696 fi
697 ''}
698 if [ ! -f ${tokenFile} ] || [ ! -s ${tokenFile} ]; then
699 printf '%s\n' ${missingToken} >&2
700 exit 1
701 fi
702 ${registerCommand}
703 ''
704 )
705 cfg.mill.executors);
706
707 executorServices =
708 mapAttrs' (
709 name: executor: let
710 service = executor.localService;
711 in
712 nameValuePair (executorServiceName name) {
713 description = "spindle mill executor ${name}";
714 after = [
715 "network.target"
716 "docker.service"
717 "spindle.service"
718 "spindle-mill-executors.service"
719 ];
720 requires = [
721 "docker.service"
722 "spindle-mill-executors.service"
723 ];
724 wantedBy = ["multi-user.target"];
725 path = deps;
726 serviceConfig = {
727 LogsDirectory = service.stateDirectory;
728 StateDirectory = service.stateDirectory;
729 Delegate = service.pipelines.microvm.cgroup.enable;
730 EnvironmentFile = mkIf (service.environmentFile != null) service.environmentFile;
731 Environment = localExecutorEnvironment executor;
732 LoadCredential = "mill-token:${executor.tokenFile}";
733 ExecStart = executorExecStart (executorServiceName name) service.package;
734 TimeoutStopSec = cfg.mill.drainTimeout + 510;
735 Restart = "always";
736 };
737 }
738 )
739 localExecutors;
740 in
741 mkIf cfg.enable {
742 assertions =
743 mapAttrsToList (name: executor: {
744 assertion = !hasPrefix "/nix/store/" executor.tokenFile;
745 message = "services.tangled.spindle.mill.executors.${name}.tokenFile must not point into the Nix store";
746 })
747 cfg.mill.executors
748 ++ mapAttrsToList (name: executor: {
749 assertion = hasPrefix "/" executor.tokenFile;
750 message = "services.tangled.spindle.mill.executors.${name}.tokenFile must be an absolute path";
751 })
752 cfg.mill.executors
753 ++ mapAttrsToList (name: executor: {
754 assertion =
755 executor.localService
756 == null
757 || !executor.localService.generateToken
758 || (!hasPrefix "/run/" executor.tokenFile && !hasPrefix "/tmp/" executor.tokenFile);
759 message = "services.tangled.spindle.mill.executors.${name}.tokenFile must persist when localService.generateToken is enabled";
760 })
761 cfg.mill.executors
762 ++ [
763 {
764 assertion = (cfg.role == "executor") == (cfg.executor != null);
765 message = "services.tangled.spindle.executor must be set exactly when role is executor";
766 }
767 {
768 assertion = cfg.mill.executors == {} || cfg.role == "mill";
769 message = "services.tangled.spindle.mill.executors requires role mill";
770 }
771 {
772 assertion = cfg.executor == null || !hasPrefix "/nix/store/" cfg.executor.tokenFile;
773 message = "services.tangled.spindle.executor.tokenFile must not point into the Nix store";
774 }
775 {
776 assertion = cfg.executor == null || hasPrefix "/" cfg.executor.tokenFile;
777 message = "services.tangled.spindle.executor.tokenFile must be an absolute path";
778 }
779 {
780 assertion = length localAgentPorts == length (unique localAgentPorts);
781 message = "services.tangled.spindle.mill.executors must use different localService.pipelines.microvm.agentPort values";
782 }
783 {
784 assertion = all (port: port != cfg.pipelines.microvm.agentPort) localAgentPorts;
785 message = "services.tangled.spindle.mill.executors must not use the main spindle's pipelines.microvm.agentPort";
786 }
787 {
788 assertion = length localStateDirectories == length (unique localStateDirectories);
789 message = "services.tangled.spindle.mill.executors must use different localService state directories";
790 }
791 {
792 assertion = !elem "spindle" localStateDirectories;
793 message = "services.tangled.spindle.mill.executors must not use the main spindle's state directory";
794 }
795 {
796 assertion = length localDbPaths == length (unique localDbPaths);
797 message = "services.tangled.spindle.mill.executors must use different localService.server.dbPath values";
798 }
799 {
800 assertion = !elem (toString cfg.server.dbPath) localDbPaths;
801 message = "services.tangled.spindle.mill.executors must not use the main spindle's server.dbPath";
802 }
803 {
804 assertion = length localRepoDirs == length (unique localRepoDirs);
805 message = "services.tangled.spindle.mill.executors must use different localService.server.repoDir values";
806 }
807 {
808 assertion = !elem (toString cfg.server.repoDir) localRepoDirs;
809 message = "services.tangled.spindle.mill.executors must not use the main spindle's server.repoDir";
810 }
811 {
812 assertion = length localDebugSshListenAddrs == length (unique localDebugSshListenAddrs);
813 message = "services.tangled.spindle.mill.executors must use different localService.pipelines.microvm.debugSsh.listenAddr values";
814 }
815 {
816 assertion =
817 if cfg.mill.artifactStore == "disk"
818 then all (service: toString service.artifactStores.disk.dir == toString cfg.artifactStores.disk.dir) (attrValues localServices)
819 else
820 all (
821 service:
822 service.artifactStores.s3.bucket
823 == cfg.artifactStores.s3.bucket
824 && service.artifactStores.s3.region == cfg.artifactStores.s3.region
825 )
826 (attrValues localServices);
827 message = "services.tangled.spindle.mill.executors must use the main spindle's configured artifact store";
828 }
829 ];
830
831 services.tangled.spindle.server = mkIf isDedicatedExecutor {
832 listenAddr = mkDefault "127.0.0.1:0";
833 hostname = mkForce cfg.executor.name;
834 owner = mkForce "";
835 };
836
837 environment.systemPackages = [
838 (pkgs.writeShellScriptBin "spindle" ''
839 export PATH="${lib.makeBinPath deps}:$PATH"
840 ${lib.optionalString (cfg.environmentFile != null) "set -a; source ${cfg.environmentFile}; set +a"}
841 ${lib.concatMapStringsSep "\n" (entry: "export ${entry}") mainEnvironment}
842 ${lib.optionalString isDedicatedExecutor "export SPINDLE_MILL_SHARED_SECRET=\"$(${pkgs.coreutils}/bin/cat ${escapeShellArg cfg.executor.tokenFile})\""}
843 exec ${cfg.package}/bin/spindle "$@"
844 '')
845 ];
846
847 virtualisation.docker.enable = true;
848
849 systemd.services =
850 {
851 spindle-mill-executors = mkIf hasRegistrations {
852 description = "register spindle mill executors";
853 before =
854 ["spindle.service"]
855 ++ mapAttrsToList (name: _: "${executorServiceName name}.service") localExecutors;
856 wantedBy = ["multi-user.target"];
857 serviceConfig = {
858 Type = "oneshot";
859 RemainAfterExit = true;
860 StateDirectory = "spindle";
861 };
862 script = registrationCommands;
863 };
864
865 spindle = {
866 description = "spindle service";
867 after =
868 [
869 "network.target"
870 "docker.service"
871 ]
872 ++ optional hasRegistrations "spindle-mill-executors.service";
873 requires = optional hasRegistrations "spindle-mill-executors.service";
874 wantedBy = ["multi-user.target"];
875 path = deps;
876 serviceConfig = {
877 LogsDirectory = "spindle";
878 StateDirectory = "spindle";
879 Delegate = cfg.pipelines.microvm.cgroup.enable;
880 EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile;
881 Environment = mainEnvironment;
882 LoadCredential = mkIf isDedicatedExecutor "mill-token:${cfg.executor.tokenFile}";
883 ExecStart =
884 if isDedicatedExecutor
885 then executorExecStart "spindle-executor" cfg.package
886 else "${pkgs.coreutils}/bin/env SPINDLE_ROLE=${cfg.role} SPINDLE_MILL_ARTIFACT_STORE=${cfg.mill.artifactStore} SPINDLE_MILL_DRAIN_TIMEOUT=${toString cfg.mill.drainTimeout}s ${cfg.package}/bin/spindle";
887 TimeoutStopSec = cfg.mill.drainTimeout + 510;
888 Restart = "always";
889 };
890 };
891 }
892 // executorServices;
893 };
894 }