From 250a077569ba93fdeb4b2459628b083c7a057ed9 Mon Sep 17 00:00:00 2001 From: Nicolas Lenz <nicolas@eisfunke.com> Date: Tue, 20 Aug 2024 00:08:20 +0200 Subject: [PATCH] wip --- home/apps/ssh.nix | 3 + home/desktop/base.nix | 13 ++- home/desktop/workspaces.nix | 2 +- nixos/extra.nix | 33 ++++++- nixos/kodi.nix | 3 + nixos/server/age.nix | 5 ++ nixos/server/default.nix | 1 + nixos/server/gitlab-runner.nix | 94 ++++++++++++++++++++ nixos/users.nix | 16 ++++ res/secrets/server/gitlab-runner-podman.age | Bin 0 -> 624 bytes secrets.nix | 1 + 11 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 nixos/server/gitlab-runner.nix create mode 100644 res/secrets/server/gitlab-runner-podman.age diff --git a/home/apps/ssh.nix b/home/apps/ssh.nix index 0d11e63f..4e849caf 100644 --- a/home/apps/ssh.nix +++ b/home/apps/ssh.nix @@ -73,6 +73,9 @@ Sets up SSH user config. # GitHub github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl + # GitLab (https://docs.gitlab.com/ee/user/gitlab_com/index.html#ssh-known_hosts-entries) + gitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf + # FachschaftenGit gitlab.fachschaften.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMK5HFOCvLwU139LPzqW8E1WP8OXj7PcPkIUhqhCLF8l diff --git a/home/desktop/base.nix b/home/desktop/base.nix index e68e4d42..8d87b9fc 100644 --- a/home/desktop/base.nix +++ b/home/desktop/base.nix @@ -73,7 +73,18 @@ lib.mkIf (!config.eisfunke.headless) { # autostart apps { command = "element-desktop"; } { command = "thunderbird"; } - { command = "vivaldi --class=vivaldi-startup"; } + + /* + Vivaldi supports setting a window class / app_id via `--class`, so I could use an assign + and a special app_id to have a specific Vivaldi window opening in the desired workspace. + + However, this app_id is reused for children windows created by e.g. ctrl+n, which means that + those would also appear in that workspace, even if the parent window is currently somewhere + else, which is annoying. + + So instead, I use this cursed workaround to move the window after startup. + */ + { command = "vivaldi && sleep 3 && swaymsg '[app_id=\"vivaldi-stable\"] move workspace 6'"; } /* I want this repo to be opened on a specific workspace. However, vscode doesn't have a way diff --git a/home/desktop/workspaces.nix b/home/desktop/workspaces.nix index 4eac4b7e..b9b72013 100644 --- a/home/desktop/workspaces.nix +++ b/home/desktop/workspaces.nix @@ -41,7 +41,7 @@ "0" = [ { app_id = "YouTube Music"; } ]; "1" = [ { app_id = "Element"; } ]; "2" = [ { app_id = "thunderbird"; } ]; - "6" = [ { app_id = "vivaldi-startup"; } ]; + # some autostarted apps are moved to a workspace by a script instead, see ./base.nix }; }; diff --git a/nixos/extra.nix b/nixos/extra.nix index d116ea47..63a97c20 100644 --- a/nixos/extra.nix +++ b/nixos/extra.nix @@ -4,7 +4,15 @@ misc stuff that's not required for a basic system { pkgs, inputs', lib, config, ... }: -lib.mkIf (!config.eisfunke.minimal) { +let + runscRootless = pkgs.writeShellApplication { + name = "runsc-rootless"; + runtimeInputs = [ pkgs.gvisor ]; + text = '' + exec runsc -ignore-cgroups "$@" + ''; + }; +in lib.mkIf (!config.eisfunke.minimal) { boot.binfmt.emulatedSystems = [ "wasm32-wasi" "wasm64-wasi" @@ -17,9 +25,32 @@ lib.mkIf (!config.eisfunke.minimal) { dns_enabled = true; network_interface = "br-podman"; # bridge device interface name }; + /* + - runs `podman system prune -f` weekly, pruning dangling data + - dangling images: untagged images + - untagged images are created e.g. when pulling a new xyz:latest image + - the previous "latest" image is now untagged, if it's unused it's dangling and will be pruned + - still tagged unused images won't be removed automatically + - manually run `podman system prune --all` for that + */ autoPrune.enable = true; + /* + - gVisor provides an alternative container runtime `runsc` + - provides better isolation + - TODO how + - I use this for GitLab Runner CI containers + - TODO grist + - can be used rootless + - `podman run -it --rm --runtime runsc --runtime-flag ignore-cgroups public.ecr.aws/docker/library/alpine:latest` + - see also https://github.com/google/gvisor/issues/311 + */ + extraPackages = [ + pkgs.gvisor + ]; }; + environment.systemPackages = [ runscRootless ]; + # bluetooth connections can be managed statefully with `bluetoothctl` hardware.bluetooth = { enable = true; diff --git a/nixos/kodi.nix b/nixos/kodi.nix index 431fdd5e..5e81aba3 100644 --- a/nixos/kodi.nix +++ b/nixos/kodi.nix @@ -10,9 +10,12 @@ let ]); in lib.mkIf (config.networking.hostName == "amethyst") { # create Kodi user + # TODO oops uid clash users.users.kodi = { isNormalUser = true; extraGroups = [ "video" ]; # for access to HDMI CEC device + uid = 1002; + autoSubUidGidRange = false; # doesn't need them }; services = { diff --git a/nixos/server/age.nix b/nixos/server/age.nix index e77ac12d..f0e72152 100644 --- a/nixos/server/age.nix +++ b/nixos/server/age.nix @@ -259,5 +259,10 @@ Agenix secrets only used in the server config are defined here instead of the to owner = "git"; group = "git"; }; + server-gitlab-runner-podman = { + file = secretsPath + /server/gitlab-runner-podman.age; + owner = "root"; + group = "root"; + }; }; } diff --git a/nixos/server/default.nix b/nixos/server/default.nix index d2eba41a..4ad497ec 100644 --- a/nixos/server/default.nix +++ b/nixos/server/default.nix @@ -21,6 +21,7 @@ modules for my services, only used on sapphire, my homeserver ./gallery.nix ./gca4hpx.nix ./git.nix + ./gitlab-runner.nix ./issuebot.nix ./lists.nix ./literature.nix diff --git a/nixos/server/gitlab-runner.nix b/nixos/server/gitlab-runner.nix new file mode 100644 index 00000000..2106e591 --- /dev/null +++ b/nixos/server/gitlab-runner.nix @@ -0,0 +1,94 @@ +{ lib, config, ... }: + +{ + # https://github.com/NixOS/nixpkgs/blob/c3aa7b8938b17aebd2deecf7be0636000d62a2b9/nixos/modules/services/continuous-integration/gitlab-runner.nix#L730 + /*users = { + users.gitlab-runner = { + group = "gitlab-runner"; + uid = config.ids.uids.gitlab-runner; # TODO was used before dynamicuser + # TODO https://docs.gitlab.com/runner/executors/docker.html#use-podman-to-run-docker-commands + linger = true; + # set subuids/subgids for rootless podman use, hardcode ranges to ensure determinism + subUidRanges = [ + { startUid = 100000; count = 65536; } + ]; + subGidRanges = [ + { startGid = 100000; count = 65536; } + ]; + }; + groups.gitlab-runner.gid = config.ids.gids.gitlab-runner; + };*/ + + systemd.services.gitlab-runner = { + # add the podman socket as prerequisite + after = [ "podman.socket" ]; + requires = [ "podman.socket" ]; + serviceConfig = { + /* + - turning off DynamicUser will run the runner service as root + - so it can use rootful podman for gVisor support + - while `runsc` *can* be used rootless with some workaraounds, I don't do that + - rootless gVisor isn't really documented + - doesn't seem "officially" supported + - can't use cgroups with it + - so for the runners I prefer rootful for "proper" gVisor + */ + #DynamicUser = lib.mkForce false; + #User = lib.mkForce "gitlab-runner"; + #Group = lib.mkForce "gitlab-runner"; + /* + - add the gitlab-runner service to the podman group for access to the socket + - note that similarly to the docker group this is root-equivalent + - I still prefer this over running the unit as root for hardening + */ + SupplementaryGroups = [ "podman" ]; + }; + }; + + environment.persistence."/persist".directories = [ "/var/lib/private/gitlab-runner" ]; + + /* + The gitlab-runner module enables Docker if there are any runners with the docker executor. As I + want to use podman instead, I have to manually force it disabled and set some workaround + options for podman support. + + TODO: PR for gitlab-runner module to support using podman out-of-the-box + */ + virtualisation.docker.enable = lib.mkForce false; + + services.gitlab-runner = { + enable = true; + # give runners some time to finish up on termination of the runner service + gracefulTermination = true; + gracefulTimeout = "1min 30s"; + settings = { + concurrent = 2; # limits concurrent jobs across *all* runners + session_server = { + listen_address = "[::]:61035"; + advertise_address = "[::]:61035"; + }; + }; + services.default = { + # TODO pull always? + description = "sapphire-podman"; + executor = "docker"; + # sets CI_SERVER_URL and CI_SERVER_TOKEN + authenticationTokenConfigFile = config.age.secrets.server-gitlab-runner-podman.path; + dockerImage = "public.ecr.aws/docker/library/alpine:latest"; + registrationFlags = [ + /* + - we can just use the podman socket here + - no need for `virtualisation.dockerSocket.podman.dockerSocket.enable` + - that just symlinks the podman socket at /run/docker.sock + */ + "--docker-host unix:///run/podman/podman.sock" + /* + TODO this is ignored?? + */ + "--docker-runtime runsc" + ]; + #dockerVolumes = [ "/cache" ]; TODO + # seems to be default anyway... + }; + }; +} diff --git a/nixos/users.nix b/nixos/users.nix index 850d3486..bde0ddd9 100644 --- a/nixos/users.nix +++ b/nixos/users.nix @@ -12,7 +12,15 @@ config for users and home-manager groups.users.gid = 100; users = { eisfunke = { + # hardcode ids to ensure determinism uid = 1000; + subUidRanges = [ + { startUid = 231072; count = 65536; } + ]; + subGidRanges = [ + { startGid = 231072; count = 65536; } + ]; + description = "Nicolas Lenz"; isNormalUser = true; extraGroups = [ @@ -35,6 +43,14 @@ config for users and home-manager privileges. */ deploy = { + # hardcode ids to ensure determinism + uid = 1001; + subUidRanges = [ + { startUid = 165536; count = 65536; } + ]; + subGidRanges = [ + { startGid = 165536; count = 65536; } + ]; isNormalUser = true; extraGroups = [ "wheel" ]; openssh.authorizedKeys.keys = config.users.users.eisfunke.openssh.authorizedKeys.keys; diff --git a/res/secrets/server/gitlab-runner-podman.age b/res/secrets/server/gitlab-runner-podman.age new file mode 100644 index 0000000000000000000000000000000000000000..14e82e16678b4ec55cb776ad6a2256875496245d GIT binary patch literal 624 zcmYdHPt{G$OD?J`D9Oyv)5|YP*Do{V(zR14F3!+RO))YxHMCU7^N2JIDOYf}^l%Ba zbj-~1PpYga$}%mr2n-5MD)R9x%6Bbw&MeC`Ep|0E3Doy;N#@E8b@mJ|$nbXYC{Htu z%BaW+F*o;f4KHx^bT%yxGAyX@3@)}X_RJ4S^F_BUIXE=SB~YQ%H6zWyz0$KH*T^x% zIorE5GbG8!#4@iqKf}?~!at<M)iEL|B-_Nj!ja3_J=MKb-^)9xAU7n)+|k7`Bq!Y0 z$=fR`GdQh0xjZ;OxhTprJKUu-(GlG?*Fx_oOGgE_^x$A$bHAd35bwzJ%rfJkAbkUG z7oS|?Di4#?EO$doC*!j6L>KK6LszcEFrV@ik4*2Vz!d%9<ihNXa^KuglRPsMAMIi@ zw-9sfO3#qc#PA5CpmKEEDttoCvjP<?&AiJ!JPX{6_1)4dgNv)m!%WN*y*-T#ioFek zBTal<bKOi6tDL>_!VI`9^L&ep0?bTG!jjVrj0!C?62tw>i@fwbE8Wt(%u2meJR|jU z^1O|b%uTp-b#)c|3r#EhjQ#zMBD@l_%=`?h%ssQ5EHd-F(gIQ<&BH27vMp1Jlf2C$ zi%qy5u^H#zT-aE+L{#T`O~#YB$-fV}O+1(QZuybF`Innw3>qdpHkrS4gW&1a=gv>? z+Hp9c?$GVsnz^k#Dn5d#j|@x|cWv$O+?VzC>F>Q$rzcNuW-XX@=V7<Yu23l-fs^Gn YR`=(NaLFrKFdJXyFcs;I`V;;L0H9OZssI20 literal 0 HcmV?d00001 diff --git a/secrets.nix b/secrets.nix index e88b1d46..078c935c 100644 --- a/secrets.nix +++ b/secrets.nix @@ -155,6 +155,7 @@ in { "res/secrets/server/gitlab-omni-eisfunkeauth.age".publicKeys = keySets.server; "res/secrets/server/gitlab-pages.age".publicKeys = keySets.server; "res/secrets/server/gitlab-pages-client.age".publicKeys = keySets.server; + "res/secrets/server/gitlab-runner-podman.age".publicKeys = keySets.server; /* Secrets used by the CI runner microvm. -- GitLab