add GH-Actions based CI with cachix

This commit is contained in:
wucke13
2023-06-09 23:43:39 +02:00
parent 73a8489232
commit 11d60bcced
4 changed files with 467 additions and 95 deletions

183
.ci/gen-workflow-files.nu Executable file
View File

@@ -0,0 +1,183 @@
#!/usr/bin/env nu
use log *
cd (git rev-parse --show-toplevel)
# map from nixos system to github runner type
let systems_map = {
# aarch64-darwin
# aarch64-linux
# TODO this one can be enabled once a oqs-sys with liboqs 0.8 is released, this can be enabled
# i686-linux: ubuntu-latest,
x86_64-darwin: macos-latest,
x86_64-linux: ubuntu-latest
}
let targets = (nix eval --json ".#packages" --apply builtins.attrNames
| from json
| par-each {|system| {
$system : (
nix eval --json $".#packages.($system)" --apply builtins.attrNames | from json
)
} }
| reduce {|it, acc| $acc | merge $it }
)
mut cachix_workflow = {
name: "Nix",
permissions: {contents: write},
on: {
pull_request: null,
push: {branches: [main]}
},
jobs: {},
}
mut release_workflow = {
name: "Release",
permissions: {contents: write},
on: { push: {tags: ["v*"]} },
jobs: {},
}
let runner_setup = [
{
uses: "actions/checkout@v3"
}
{
uses: "cachix/install-nix-action@v21",
with: { nix_path: "nixpkgs=channel:nixos-unstable" }
}
{
uses: "cachix/cachix-action@v12",
with: {
name: rosenpass,
authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}"
}
}
]
for system in ($targets | columns) {
if ($systems_map | get -i $system | is-empty) {
log info $"skipping ($system), since there are no GH-Actions runners for it"
continue
}
# lookup the correct runner for $system
let runs_on = [ ($systems_map | get $system) ]
# add jobs for all derivations
let derivations = ($targets | get $system)
for derivation in $derivations {
# skip the default derivation, its an alias of the rosenpass derivation
if ($derivation == "default") {
continue
}
# job_id for GH-Actions
let id = $"($system)---($derivation)"
# name displayed
let name = $"($system).($derivation)"
# collection of dependencies
mut needs = []
if ($derivation | str ends-with "oci-image") {
$needs = ($needs | append ( $derivation | str replace '(.+)-oci-image' "$1" ))
}
if ($derivation == "proof-proverif") {
$needs = ($needs | append "proverif-patched")
}
if ($derivation == "release-package") {
$needs = ($needs | append ($derivations | find "rosenpass"))
}
# prefix all needs with the system to get a full job_id
$needs = ($needs | each {|drv| $"($system)---($drv)"})
mut new_job = {
name: $"Build ($name)",
"runs-on": $runs_on,
needs: $needs,
steps: ($runner_setup | append [
{
name: Build,
run: $"nix build .#packages.($system).($derivation) --print-build-logs"
}
])
}
$cachix_workflow.jobs = ($cachix_workflow.jobs | insert $id $new_job )
}
# add check job
$cachix_workflow.jobs = ($cachix_workflow.jobs | insert $"($system)---check" {
name: $"Run Nix checks on ($system)",
"runs-on": $runs_on,
steps: ($runner_setup | append {
name: Check,
run: "nix flake check . --print-build-logs"
})
})
# add release job
$release_workflow.jobs = ($release_workflow.jobs | insert $"($system)---release" {
name: $"Build release artifacts for ($system)",
"runs-on": $runs_on,
steps: ($runner_setup | append [
{
name: "Build release",
run: "nix build .#release-package --print-build-logs"
}
{
name: Release,
uses: "softprops/action-gh-release@v1",
with: {
draft: "${{ contains(github.ref_name, 'rc') }}",
prerelease: "${{ contains(github.ref_name, 'alpha') || contains(github.ref_name, 'beta') }}",
files: "result/*"
}
}
])
})
}
# add whitepaper job with upload
let system = "x86_64-linux"
$cachix_workflow.jobs = ($cachix_workflow.jobs | insert $"($system)---whitepaper-upload" {
name: $"Upload whitepaper ($system)",
"runs-on": ($systems_map | get $system),
steps: ($runner_setup | append [
{
name: "Git add git sha and commit",
run: "cd papers && ./tex/gitinfo2.sh && git add gitHeadInfo.gin"
}
{
name: Build,
run: $"nix build .#packages.($system).whitepaper --print-build-logs"
}
{
name: "Deploy PDF artifacts",
"if": "${{ github.ref == 'refs/heads/main' }}",
uses: "peaceiris/actions-gh-pages@v3",
with: {
github_token: "${{ secrets.GITHUB_TOKEN }}",
publish_dir: result/,
publish_branch: papers-pdf,
force_orphan: true
}
}
])
})
log info "saving nix-cachix workflow"
$cachix_workflow | to yaml | save --force .github/workflows/nix.yaml
$release_workflow | to yaml | save --force .github/workflows/release.yaml
log info "prettify generated yaml"
prettier -w .github/workflows/

View File

@@ -2,87 +2,248 @@ name: Nix
permissions:
contents: write
on:
pull_request:
pull_request: null
push:
branches: [main]
branches:
- main
jobs:
build:
name: Build ${{ matrix.derivation }} on ${{ matrix.nix-system }}
x86_64-darwin---release-package:
name: Build x86_64-darwin.release-package
runs-on:
- nix
- ${{ matrix.nix-system }}
strategy:
fail-fast: false
matrix:
nix-system:
- x86_64-linux
- i686-linux
#- aarch64-linux -- Broken; see https://github.com/rosenpass/rosenpass/issues/62
derivation:
- rosenpass
- rosenpass-static
- rosenpass-oci-image
- rosenpass-static-oci-image
- proof-proverif
- whitepaper
exclude:
# these do not exist
- nix-system: i686-linux
derivation: proof-proverif
- nix-system: i686-linux
derivation: whitepaper
# these fail currently
# TODO enable once https://github.com/open-quantum-safe/liboqs-rust/issues/202 is fixed
- nix-system: i686-linux
derivation: rosenpass-static
- nix-system: i686-linux
derivation: rosenpass-static-oci-image
- macos-latest
needs:
- x86_64-darwin---rosenpass
- x86_64-darwin---rosenpass-oci-image
steps:
- uses: actions/checkout@v3
- name: Generate gitHeadInfo.gin for the whitepaper
if: ${{ matrix.derivation == 'whitepaper' }}
run: ( cd papers && ./tex/gitinfo2.sh && git add gitHeadInfo.gin )
- name: Build ${{ matrix.derivation }}@${{ matrix.nix-system }}
run: |
# build the package
nix build .#packages.${{ matrix.nix-system }}.${{ matrix.derivation }} --print-build-logs
# copy over the results
if [[ -f $(readlink --canonicalize result ) ]]; then
mkdir -- ${{ matrix.derivation }}-${{ matrix.nix-system }}
fi
cp --recursive -- $(readlink --canonicalize result) ${{ matrix.derivation }}-${{ matrix.nix-system }}
chmod --recursive ug+rw -- ${{ matrix.derivation }}-${{ matrix.nix-system }}
# add version information
git rev-parse --abbrev-ref HEAD > ${{ matrix.derivation }}-${{ matrix.nix-system }}/git-version
git rev-parse HEAD > ${{ matrix.derivation }}-${{ matrix.nix-system }}/git-sha
# override the `rp` script to keep compatible with non-nix systems
if [[ -f ${{ matrix.derivation }}-${{ matrix.nix-system }}/bin/rp ]]; then
cp --force -- rp ${{ matrix.derivation }}-${{ matrix.nix-system }}/bin/
fi
- name: Upload build results
uses: actions/upload-artifact@v3
- uses: cachix/install-nix-action@v21
with:
name: ${{ matrix.derivation }}-${{ matrix.nix-system }}
path: ${{ matrix.derivation }}-${{ matrix.nix-system }}
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build
run: nix build .#packages.x86_64-darwin.release-package --print-build-logs
x86_64-darwin---rosenpass:
name: Build x86_64-darwin.rosenpass
runs-on:
- macos-latest
needs: []
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build
run: nix build .#packages.x86_64-darwin.rosenpass --print-build-logs
x86_64-darwin---rosenpass-oci-image:
name: Build x86_64-darwin.rosenpass-oci-image
runs-on:
- macos-latest
needs:
- x86_64-darwin---rosenpass
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build
run: nix build .#packages.x86_64-darwin.rosenpass-oci-image --print-build-logs
x86_64-darwin---check:
name: Run Nix checks on x86_64-darwin
runs-on:
- macos-latest
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Check
run: nix flake check . --print-build-logs
x86_64-linux---proof-proverif:
name: Build x86_64-linux.proof-proverif
runs-on:
- ubuntu-latest
needs:
- x86_64-linux---proverif-patched
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build
run: nix build .#packages.x86_64-linux.proof-proverif --print-build-logs
x86_64-linux---proverif-patched:
name: Build x86_64-linux.proverif-patched
runs-on:
- ubuntu-latest
needs: []
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build
run: nix build .#packages.x86_64-linux.proverif-patched --print-build-logs
x86_64-linux---release-package:
name: Build x86_64-linux.release-package
runs-on:
- ubuntu-latest
needs:
- x86_64-linux---rosenpass
- x86_64-linux---rosenpass-oci-image
- x86_64-linux---rosenpass-static
- x86_64-linux---rosenpass-static-oci-image
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build
run: nix build .#packages.x86_64-linux.release-package --print-build-logs
x86_64-linux---rosenpass:
name: Build x86_64-linux.rosenpass
runs-on:
- ubuntu-latest
needs: []
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build
run: nix build .#packages.x86_64-linux.rosenpass --print-build-logs
x86_64-linux---rosenpass-oci-image:
name: Build x86_64-linux.rosenpass-oci-image
runs-on:
- ubuntu-latest
needs:
- x86_64-linux---rosenpass
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build
run: nix build .#packages.x86_64-linux.rosenpass-oci-image --print-build-logs
x86_64-linux---rosenpass-static:
name: Build x86_64-linux.rosenpass-static
runs-on:
- ubuntu-latest
needs: []
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build
run: nix build .#packages.x86_64-linux.rosenpass-static --print-build-logs
x86_64-linux---rosenpass-static-oci-image:
name: Build x86_64-linux.rosenpass-static-oci-image
runs-on:
- ubuntu-latest
needs:
- x86_64-linux---rosenpass-static
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build
run: nix build .#packages.x86_64-linux.rosenpass-static-oci-image --print-build-logs
x86_64-linux---whitepaper:
name: Build x86_64-linux.whitepaper
runs-on:
- ubuntu-latest
needs: []
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build
run: nix build .#packages.x86_64-linux.whitepaper --print-build-logs
x86_64-linux---check:
name: Run Nix checks on x86_64-linux
runs-on:
- ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Check
run: nix flake check . --print-build-logs
x86_64-linux---whitepaper-upload:
name: Upload whitepaper x86_64-linux
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Git add git sha and commit
run: cd papers && ./tex/gitinfo2.sh && git add gitHeadInfo.gin
- name: Build
run: nix build .#packages.x86_64-linux.whitepaper --print-build-logs
- name: Deploy PDF artifacts
if: ${{ matrix.derivation == 'whitepaper' && github.ref == 'refs/heads/main' }}
if: ${{ github.ref == 'refs/heads/main' }}
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ${{ matrix.derivation }}-${{ matrix.nix-system }}
publish_dir: result/
publish_branch: papers-pdf
force_orphan: true
checks:
name: Run Nix checks
runs-on: nixos
needs: build
steps:
- uses: actions/checkout@v3
- name: Run Checks
run: nix flake check . --print-build-logs

View File

@@ -3,28 +3,48 @@ permissions:
contents: write
on:
push:
tags: ["v*"]
tags:
- v*
jobs:
release:
name: Release for ${{ matrix.nix-system }}
x86_64-darwin---release:
name: Build release artifacts for x86_64-darwin
runs-on:
- nix
- ${{ matrix.nix-system }}
strategy:
fail-fast: false
matrix:
nix-system:
- x86_64-linux
#- aarch64-linux -- Broken; see https://github.com/rosenpass/rosenpass/issues/62
- macos-latest
steps:
- uses: actions/checkout@v3
- name: Build release-package for ${{ matrix.nix-system }}
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build release
run: nix build .#release-package --print-build-logs
- name: Release
uses: softprops/action-gh-release@v1
with:
draft: ${{ contains(github.ref_name, 'rc') }}
prerelease: ${{ contains(github.ref_name, 'alpha') || contains(github.ref_name, 'beta') }}
files: |
result/*
files: result/*
x86_64-linux---release:
name: Build release artifacts for x86_64-linux
runs-on:
- ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v21
with:
nix_path: nixpkgs=channel:nixos-unstable
- uses: cachix/cachix-action@v12
with:
name: rosenpass
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build release
run: nix build .#release-package --print-build-logs
- name: Release
uses: softprops/action-gh-release@v1
with:
draft: ${{ contains(github.ref_name, 'rc') }}
prerelease: ${{ contains(github.ref_name, 'alpha') || contains(github.ref_name, 'beta') }}
files: result/*

View File

@@ -99,6 +99,12 @@
cargo = toolchain;
rustc = toolchain;
};
# used to trick the build.rs into believing that CMake was ran **again**
fakecmake = pkgs.writeScriptBin "cmake" ''
#! ${pkgs.stdenv.shell} -e
true
'';
in
naersk.buildPackage
{
@@ -134,15 +140,17 @@
};
overrideMain = x: {
# CMake detects that it was served a _foreign_ target dir, thus we have to
# convice it a little
# TODO this still re-builds liboqs in the second step, which is wasteful
preBuild = x.preBuild + ''
find -name CMakeCache.txt -exec sed s_/dummy-src/_/source/_g --in-place {} \;
'' + (lib.optionalString isStatic ''
# CMake detects that it was served a _foreign_ target dir, and CMake
# would be executed again upon the second build step of naersk.
# By adding our specially optimized CMake version, we reduce the cost
# of recompilation by 99 % while, while avoiding any CMake errors.
nativeBuildInputs = [ (lib.hiPrio fakecmake) ] ++ x.nativeBuildInputs;
# make sure that libc is linked, under musl this is not the case per
# default
preBuild = (lib.optionalString isStatic ''
NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -lc"
'')
;
'');
preInstall = ''
install -D ${./rp} $out/bin/rp