Allow compiling to assembly and friends in the orchestrated Docker container

.github/workflows/ci.yml

@@ -185,7 +185,7 @@ jobs:
       run: cargo fmt --manifest-path top-crates/Cargo.toml --check
     - name: Build backend
       run: |-
-        mkdir -p ui/target; docker run --rm -v $PWD/ui:/ui -v ~/.cargo/git:/root/.cargo/git -v ~/.cargo/registry:/root/.cargo/registry --workdir /ui rust:alpine sh -c '
+        mkdir -p ui/target; docker run --rm -v $PWD/compiler/base/asm-cleanup:/compiler/base/asm-cleanup -v $PWD/compiler/base/orchestrator:/compiler/base/orchestrator -v $PWD/compiler/base/modify-cargo-toml:/compiler/base/modify-cargo-toml -v $PWD/ui:/ui -v ~/.cargo/git:/root/.cargo/git -v ~/.cargo/registry:/root/.cargo/registry --workdir /ui rust:alpine sh -c '
           apk add musl-dev openssl-dev openssl-libs-static
 
           # Adding -C relocation-model=static due to

ci/workflows.yml

@@ -288,6 +288,9 @@ workflows:
                 docker
                 run
                 --rm
+                -v $PWD/compiler/base/asm-cleanup:/compiler/base/asm-cleanup
+                -v $PWD/compiler/base/orchestrator:/compiler/base/orchestrator
+                -v $PWD/compiler/base/modify-cargo-toml:/compiler/base/modify-cargo-toml
                 -v $PWD/ui:/ui
                 -v ~/.cargo/git:/root/.cargo/git
                 -v ~/.cargo/registry:/root/.cargo/registry

compiler/base/.dockerignore

@@ -1 +1,3 @@
+asm-cleanup/target
 modify-cargo-toml/target
+orchestrator/target

compiler/base/Dockerfile

@@ -61,11 +61,41 @@ FROM bare-sources as munge
 ADD --chown=playground modify-cargo-toml /playground/modify-cargo-toml
 RUN cargo build --release --manifest-path=/playground/modify-cargo-toml/Cargo.toml
 
+# Set up cargo-chef for faster builds
+
+FROM bare-sources as chef-available
+
+RUN cargo install cargo-chef
+
+WORKDIR /orchestrator
+
+# Prepare the orchestrator's dependencies
+
+FROM chef-available as prepare-orchestrator
+
+COPY --chown=playground asm-cleanup /asm-cleanup
+COPY --chown=playground modify-cargo-toml /modify-cargo-toml
+COPY --chown=playground orchestrator /orchestrator
+RUN cargo chef prepare
+
+# Build the orchestrator
+
+FROM chef-available as build-orchestrator
+
+COPY --chown=playground asm-cleanup /asm-cleanup
+COPY --chown=playground modify-cargo-toml /modify-cargo-toml
+COPY --chown=playground --from=prepare-orchestrator /orchestrator/recipe.json /orchestrator/recipe.json
+RUN cargo chef cook --release
+
+COPY --chown=playground orchestrator /orchestrator
+RUN cargo install --path .
+
 # Compiler and sources
 
 FROM bare-sources as sources
 
 COPY --from=munge /playground/modify-cargo-toml/target/release/modify-cargo-toml /playground/.cargo/bin
+COPY --from=build-orchestrator /playground/.cargo/bin/worker /playground/.cargo/bin/worker
 
 # Compiler and pre-compiled crates
 

compiler/base/asm-cleanup/.gitignore

@@ -0,0 +1,2 @@
+Cargo.lock
+target/

compiler/base/asm-cleanup/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "asm-cleanup"
+version = "0.1.0"
+edition = "2018"
+
+[workspace]
+
+[dependencies]
+lazy_static = "1.0.0"
+petgraph = "0.6.0"
+regex = "1.0.0"
+rustc-demangle = "0.1.5"

compiler/base/cargo-wasm

@@ -41,4 +41,9 @@ for wasm in $(find target/ -name '*wasm' -not -path '*/deps/*'); do
     # wasm2wat spits out an error that we don't care about, so hide it
     # https://github.com/WebAssembly/wabt/issues/842
     # https://stackoverflow.com/a/15936384/155423
+
+    # The streaming playground expects the file to be without the
+    # extension while the original playground expects it to be with
+    # the extension. Support both for now.
+    cp "${output}.wat" "${output}"
 done

compiler/base/entrypoint.sh

@@ -2,10 +2,14 @@
 
 set -eu
 
-timeout=${PLAYGROUND_TIMEOUT:-10}
+if [[ -z "${PLAYGROUND_ORCHESTRATOR:-}" ]]; then
+    timeout=${PLAYGROUND_TIMEOUT:-10}
 
-modify-cargo-toml
+    modify-cargo-toml
 
-# Don't use `exec` here. The shell is what prints out the useful
-# "Killed" message
-timeout --signal=KILL ${timeout} "$@"
+    # Don't use `exec` here. The shell is what prints out the useful
+    # "Killed" message
+    timeout --signal=KILL ${timeout} "$@"
+else
+    exec "$@"
+fi

compiler/base/modify-cargo-toml/src/lib.rs

@@ -0,0 +1,131 @@
+extern crate serde;
+#[macro_use]
+extern crate serde_derive;
+extern crate toml;
+
+use std::collections::BTreeMap;
+use toml::Value;
+
+type Other = BTreeMap<String, Value>;
+
+fn modify<F, T>(cargo_toml: Value, f: F) -> Value
+where
+    F: FnOnce(T) -> T,
+    T: serde::Serialize + for<'de> serde::Deserialize<'de>,
+{
+    let cargo_toml = cargo_toml.try_into().unwrap();
+
+    let cargo_toml = f(cargo_toml);
+
+    Value::try_from(cargo_toml).unwrap()
+}
+
+fn ensure_string_in_vec(values: &mut Vec<String>, val: &str) {
+    if !values.iter().any(|f| f == val) {
+        values.push(val.into());
+    }
+}
+
+pub fn set_edition(cargo_toml: Value, edition: &str) -> Value {
+    #[derive(Debug, Serialize, Deserialize)]
+    #[serde(rename_all = "kebab-case")]
+    struct CargoToml {
+        package: Package,
+        #[serde(flatten)]
+        other: Other,
+    }
+
+    #[derive(Debug, Serialize, Deserialize)]
+    #[serde(rename_all = "kebab-case")]
+    struct Package {
+        #[serde(default)]
+        edition: String,
+        #[serde(flatten)]
+        other: Other,
+    }
+
+    modify(cargo_toml, |mut cargo_toml: CargoToml| {
+        cargo_toml.package.edition = edition.into();
+        cargo_toml
+    })
+}
+
+pub fn remove_dependencies(cargo_toml: Value) -> Value {
+    #[derive(Debug, Serialize, Deserialize)]
+    #[serde(rename_all = "kebab-case")]
+    struct CargoToml {
+        dependencies: BTreeMap<String, Value>,
+        #[serde(flatten)]
+        other: Other,
+    }
+
+    modify(cargo_toml, |mut cargo_toml: CargoToml| {
+        cargo_toml.dependencies.clear();
+        cargo_toml
+    })
+}
+
+pub fn set_crate_type(cargo_toml: Value, crate_type: &str) -> Value {
+    #[derive(Debug, Serialize, Deserialize)]
+    #[serde(rename_all = "kebab-case")]
+    struct CargoToml {
+        #[serde(default)]
+        lib: Lib,
+        #[serde(flatten)]
+        other: Other,
+    }
+
+    #[derive(Debug, Default, Serialize, Deserialize)]
+    #[serde(rename_all = "kebab-case")]
+    struct Lib {
+        #[serde(default, skip_serializing_if = "Vec::is_empty")]
+        crate_type: Vec<String>,
+        #[serde(default)]
+        proc_macro: bool,
+        #[serde(flatten)]
+        other: Other,
+    }
+
+    modify(cargo_toml, |mut cargo_toml: CargoToml| {
+        if crate_type == "proc-macro" {
+            cargo_toml.lib.proc_macro = true;
+        } else {
+            ensure_string_in_vec(&mut cargo_toml.lib.crate_type, crate_type);
+        }
+        cargo_toml
+    })
+}
+
+pub fn set_release_lto(cargo_toml: Value, lto: bool) -> Value {
+    #[derive(Debug, Serialize, Deserialize)]
+    #[serde(rename_all = "kebab-case")]
+    struct CargoToml {
+        #[serde(default)]
+        profile: Profiles,
+        #[serde(flatten)]
+        other: Other,
+    }
+
+    #[derive(Debug, Default, Serialize, Deserialize)]
+    #[serde(rename_all = "kebab-case")]
+    struct Profiles {
+        #[serde(default)]
+        release: Profile,
+        #[serde(flatten)]
+        other: Other,
+    }
+
+    #[derive(Debug, Default, Serialize, Deserialize)]
+    #[serde(rename_all = "kebab-case")]
+    struct Profile {
+        #[serde(default)]
+        lto: bool,
+        #[serde(flatten)]
+        other: Other,
+    }
+
+    modify(cargo_toml, |mut cargo_toml: CargoToml| {
+        cargo_toml.profile.release.lto = lto;
+        cargo_toml
+    })
+}

compiler/base/modify-cargo-toml/src/main.rs

@@ -1,9 +1,8 @@
-extern crate serde;
-#[macro_use]
-extern crate serde_derive;
+extern crate modify_cargo_toml;
 extern crate toml;
 
-use std::{collections::BTreeMap, env, ffi::OsString, fs, path::PathBuf};
+use modify_cargo_toml::*;
+use std::{env, ffi::OsString, fs, path::PathBuf};
 use toml::Value;
 
 fn main() {
@@ -41,127 +40,3 @@ fn main() {
     fs::write(&output_filename, output)
         .unwrap_or_else(|e| panic!("Cannot write to {}: {}", output_filename.display(), e));
 }
-
-type Other = BTreeMap<String, Value>;
-
-fn modify<F, T>(cargo_toml: Value, f: F) -> Value
-where
-    F: FnOnce(T) -> T,
-    T: serde::Serialize + for<'de> serde::Deserialize<'de>,
-{
-    let cargo_toml = cargo_toml.try_into().unwrap();
-
-    let cargo_toml = f(cargo_toml);
-
-    Value::try_from(cargo_toml).unwrap()
-}
-
-fn ensure_string_in_vec(values: &mut Vec<String>, val: &str) {
-    if !values.iter().any(|f| f == val) {
-        values.push(val.into());
-    }
-}
-
-fn set_edition(cargo_toml: Value, edition: &str) -> Value {
-    #[derive(Debug, Serialize, Deserialize)]
-    #[serde(rename_all = "kebab-case")]
-    struct CargoToml {
-        package: Package,
-        #[serde(flatten)]
-        other: Other,
-    }
-
-    #[derive(Debug, Serialize, Deserialize)]
-    #[serde(rename_all = "kebab-case")]
-    struct Package {
-        #[serde(default)]
-        edition: String,
-        #[serde(flatten)]
-        other: Other,
-    }
-
-    modify(cargo_toml, |mut cargo_toml: CargoToml| {
-        cargo_toml.package.edition = edition.into();
-        cargo_toml
-    })
-}
-
-fn remove_dependencies(cargo_toml: Value) -> Value {
-    #[derive(Debug, Serialize, Deserialize)]
-    #[serde(rename_all = "kebab-case")]
-    struct CargoToml {
-        dependencies: BTreeMap<String, Value>,
-        #[serde(flatten)]
-        other: Other,
-    }
-
-    modify(cargo_toml, |mut cargo_toml: CargoToml| {
-        cargo_toml.dependencies.clear();
-        cargo_toml
-    })
-}
-
-fn set_crate_type(cargo_toml: Value, crate_type: &str) -> Value {
-    #[derive(Debug, Serialize, Deserialize)]
-    #[serde(rename_all = "kebab-case")]
-    struct CargoToml {
-        #[serde(default)]
-        lib: Lib,
-        #[serde(flatten)]
-        other: Other,
-    }
-
-    #[derive(Debug, Default, Serialize, Deserialize)]
-    #[serde(rename_all = "kebab-case")]
-    struct Lib {
-        #[serde(default, skip_serializing_if = "Vec::is_empty")]
-        crate_type: Vec<String>,
-        #[serde(default)]
-        proc_macro: bool,
-        #[serde(flatten)]
-        other: Other,
-    }
-
-    modify(cargo_toml, |mut cargo_toml: CargoToml| {
-        if crate_type == "proc-macro" {
-            cargo_toml.lib.proc_macro = true;
-        } else {
-            ensure_string_in_vec(&mut cargo_toml.lib.crate_type, crate_type);
-        }
-        cargo_toml
-    })
-}
-
-fn set_release_lto(cargo_toml: Value, lto: bool) -> Value {
-    #[derive(Debug, Serialize, Deserialize)]
-    #[serde(rename_all = "kebab-case")]
-    struct CargoToml {
-        #[serde(default)]
-        profile: Profiles,
-        #[serde(flatten)]
-        other: Other,
-    }
-
-    #[derive(Debug, Default, Serialize, Deserialize)]
-    #[serde(rename_all = "kebab-case")]
-    struct Profiles {
-        #[serde(default)]
-        release: Profile,
-        #[serde(flatten)]
-        other: Other,
-    }
-
-    #[derive(Debug, Default, Serialize, Deserialize)]
-    #[serde(rename_all = "kebab-case")]
-    struct Profile {
-        #[serde(default)]
-        lto: bool,
-        #[serde(flatten)]
-        other: Other,
-    }
-
-    modify(cargo_toml, |mut cargo_toml: CargoToml| {
-        cargo_toml.profile.release.lto = lto;
-        cargo_toml
-    })
-}

compiler/base/orchestrator/.gitignore

@@ -0,0 +1 @@
+target