Skip to content

Commit 927c55d

Browse files
committed
Auto merge of #38748 - alexcrichton:upload, r=brson
travis: Start uploading artifacts on commits This commit starts adding the infrastructure for uploading release artifacts from AppVeyor/Travis on each commit. The idea is that eventually we'll upload a full release to AppVeyor/Travis in accordance with plans [outlined earlier]. Right now this configures Travis/Appveyor to upload all tarballs in the `dist` directory, and various images are updated to actually produce tarballs in these directories. These are nowhere near ready to be actual release artifacts, but this should allow us to play around with it and test it out. Once this commit lands we should start seeing artifacts uploaded on each commit. [outlined earlier]: https://internals.rust-lang.org/t/rust-ci-release-infrastructure-changes/4489
2 parents 1a2ed98 + 3187672 commit 927c55d

File tree

19 files changed

+148
-64
lines changed

19 files changed

+148
-64
lines changed

.travis.yml

+37-9
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,20 @@ osx_image: xcode8.2
1313
matrix:
1414
include:
1515
# Linux builders, all docker images
16-
- env: IMAGE=arm-android
17-
- env: IMAGE=cross
18-
- env: IMAGE=dist-arm-unknown-linux-gnueabi
19-
- env: IMAGE=dist-x86_64-unknown-freebsd
20-
- env: IMAGE=i686-gnu
16+
- env: IMAGE=arm-android DEPLOY=1
17+
- env: IMAGE=cross DEPLOY=1
18+
- env: IMAGE=dist-arm-unknown-linux-gnueabi DEPLOY=1
19+
- env: IMAGE=dist-x86_64-unknown-freebsd DEPLOY=1
20+
- env: IMAGE=i686-gnu DEPLOY=1
2121
- env: IMAGE=i686-gnu-nopt
22-
- env: IMAGE=x86_64-gnu
22+
- env: IMAGE=x86_64-gnu DEPLOY=1
2323
- env: IMAGE=x86_64-gnu-full-bootstrap
2424
- env: IMAGE=x86_64-gnu-aux
2525
- env: IMAGE=x86_64-gnu-debug
2626
- env: IMAGE=x86_64-gnu-nopt
2727
- env: IMAGE=x86_64-gnu-make
2828
- env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1 RUST_BACKTRACE=1
29-
- env: IMAGE=x86_64-musl
29+
- env: IMAGE=x86_64-musl DEPLOY=1
3030
- env: IMAGE=x86_64-gnu-distcheck
3131

3232
# OSX builders
@@ -39,9 +39,10 @@ matrix:
3939
curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 |
4040
tar xJf - -C /usr/local/bin --strip-components=1
4141
- env: >
42-
RUST_CHECK_TARGET=check
42+
SCRIPT="./x.py test && ./x.py dist"
4343
RUST_CONFIGURE_ARGS=--build=i686-apple-darwin
4444
SRC=.
45+
DEPLOY=1
4546
os: osx
4647
install: *osx_install_sccache
4748
- env: >
@@ -51,9 +52,10 @@ matrix:
5152
os: osx
5253
install: *osx_install_sccache
5354
- env: >
54-
RUST_CHECK_TARGET=
55+
RUST_CHECK_TARGET=dist
5556
RUST_CONFIGURE_ARGS=--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios
5657
SRC=.
58+
DEPLOY=1
5759
os: osx
5860
install: *osx_install_sccache
5961
@@ -91,3 +93,29 @@ notifications:
9193
cache:
9294
directories:
9395
- $HOME/docker
96+
97+
before_deploy:
98+
- mkdir -p deploy/$TRAVIS_COMMIT
99+
- >
100+
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
101+
cp build/dist/*.tar.gz deploy/$TRAVIS_COMMIT &&
102+
find "deploy/$TRAVIS_COMMIT" -maxdepth 1 -type f -exec sh -c 'shasum -a 256 -b "{}" > "{}.sha256"' \;;
103+
else
104+
cp obj/build/dist/*.tar.gz deploy/$TRAVIS_COMMIT &&
105+
find "deploy/$TRAVIS_COMMIT" -maxdepth 1 -type f -exec sh -c 'sha256sum -b "{}" > "{}.sha256"' \;;
106+
fi
107+
108+
deploy:
109+
- provider: s3
110+
bucket: rust-lang-ci
111+
skip_cleanup: true
112+
local_dir: deploy
113+
upload_dir: rustc-builds
114+
acl: public_read
115+
region: us-east-1
116+
access_key_id: AKIAIPQVNYF2T3DTYIWQ
117+
secret_access_key:
118+
secure: "FBqDqOTeIPMu6v/WYPf4CFSlh9rLRZGKVtpLa5KkyuOhXRTrnEzBduEtS8/FMIxdQImvurhSvxWvqRybMOi4qoVfjMqqpHAI7uBbidbrvAcJoHNsx6BgUNVCIoH6a0UsAjTUtm6/YPIpzbHoLZXPL0GrHPMk6Mu04qVSmcYNWn4="
119+
on:
120+
branch: auto
121+
condition: $DEPLOY = 1

appveyor.yml

+34-4
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,12 @@ environment:
88
# 32/64 bit MSVC
99
- MSYS_BITS: 64
1010
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
11-
RUST_CHECK_TARGET: check
11+
SCRIPT: python x.py test && python x.py dist
12+
DEPLOY: 1
1213
- MSYS_BITS: 32
1314
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
14-
RUST_CHECK_TARGET: check
15+
SCRIPT: python x.py test && python x.py dist
16+
DEPLOY: 1
1517

1618
# MSVC makefiles
1719
- MSYS_BITS: 64
@@ -50,10 +52,11 @@ environment:
5052
# too long on appveyor and this is tested by rustbuild below.
5153
- MSYS_BITS: 32
5254
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
53-
RUST_CHECK_TARGET: check
55+
SCRIPT: python x.py test && python x.py dist
5456
MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
5557
MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
5658
MINGW_DIR: mingw32
59+
DEPLOY: 1
5760

5861
- MSYS_BITS: 32
5962
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --disable-rustbuild
@@ -63,11 +66,12 @@ environment:
6366
MINGW_DIR: mingw32
6467

6568
- MSYS_BITS: 64
66-
RUST_CHECK_TARGET: check
69+
SCRIPT: python x.py test && python x.py dist
6770
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
6871
MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
6972
MINGW_ARCHIVE: x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z
7073
MINGW_DIR: mingw64
74+
DEPLOY: 1
7175

7276
clone_depth: 1
7377
build: false
@@ -123,6 +127,32 @@ branches:
123127
only:
124128
- auto
125129

130+
before_deploy:
131+
- ps: |
132+
New-Item -Path deploy -ItemType directory
133+
Get-ChildItem -Path build\dist -Filter '*.tar.gz' | Move-Item -Destination deploy
134+
Get-FileHash .\deploy\* | ForEach-Object {
135+
[io.file]::WriteAllText($_.Path + ".sha256", $_.Hash.ToLower() + "`n")
136+
}
137+
Get-ChildItem -Path deploy | Foreach-Object {
138+
Push-AppveyorArtifact $_.FullName -FileName ${env:APPVEYOR_REPO_COMMIT}/$_
139+
}
140+
141+
deploy:
142+
- provider: S3
143+
skip_cleanup: true
144+
access_key_id: AKIAIPQVNYF2T3DTYIWQ
145+
secret_access_key:
146+
secure: +11jsUNFTQ9dq5Ad1i2+PeUJaXluFJ0zIJAXESE1dFT3Kdjku4/eDdgyjgsB6GnV
147+
bucket: rust-lang-ci
148+
set_public: true
149+
region: us-east-1
150+
artifact: /.*\.(tar.gz|sha256)/
151+
folder: rustc-builds
152+
on:
153+
branch: auto
154+
DEPLOY: 1
155+
126156
# init:
127157
# - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
128158
# on_finish:

src/bootstrap/check.rs

+3
Original file line numberDiff line numberDiff line change
@@ -457,6 +457,8 @@ fn krate_android(build: &Build,
457457

458458
let output = output(Command::new("adb").arg("shell").arg(&program));
459459
println!("{}", output);
460+
461+
t!(fs::create_dir_all(build.out.join("tmp")));
460462
build.run(Command::new("adb")
461463
.arg("pull")
462464
.arg(&log)
@@ -516,6 +518,7 @@ pub fn android_copy_libs(build: &Build,
516518
}
517519

518520
println!("Android copy libs to emulator ({})", target);
521+
build.run(Command::new("adb").arg("wait-for-device"));
519522
build.run(Command::new("adb").arg("remount"));
520523
build.run(Command::new("adb").args(&["shell", "rm", "-r", ADB_TEST_DIR]));
521524
build.run(Command::new("adb").args(&["shell", "mkdir", ADB_TEST_DIR]));

src/bootstrap/compile.rs

+31-19
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
//! compiler. This module is also responsible for assembling the sysroot as it
1717
//! goes along from the output of the previous stage.
1818
19-
use std::cmp;
2019
use std::collections::HashMap;
2120
use std::fs::{self, File};
2221
use std::path::{Path, PathBuf};
@@ -59,7 +58,7 @@ pub fn std(build: &Build, target: &str, compiler: &Compiler) {
5958
}
6059

6160
build.run(&mut cargo);
62-
update_mtime(&libstd_stamp(build, &compiler, target));
61+
update_mtime(build, &libstd_stamp(build, &compiler, target));
6362
}
6463

6564
/// Link all libstd rlibs/dylibs into the sysroot location.
@@ -145,7 +144,7 @@ pub fn test(build: &Build, target: &str, compiler: &Compiler) {
145144
cargo.arg("--manifest-path")
146145
.arg(build.src.join("src/rustc/test_shim/Cargo.toml"));
147146
build.run(&mut cargo);
148-
update_mtime(&libtest_stamp(build, compiler, target));
147+
update_mtime(build, &libtest_stamp(build, compiler, target));
149148
}
150149

151150
/// Same as `std_link`, only for libtest
@@ -390,26 +389,39 @@ pub fn tool(build: &Build, stage: u32, host: &str, tool: &str) {
390389
}
391390

392391
/// Updates the mtime of a stamp file if necessary, only changing it if it's
393-
/// older than some other file in the same directory.
392+
/// older than some other library file in the same directory.
394393
///
395394
/// We don't know what file Cargo is going to output (because there's a hash in
396395
/// the file name) but we know where it's going to put it. We use this helper to
397396
/// detect changes to that output file by looking at the modification time for
398397
/// all files in a directory and updating the stamp if any are newer.
399-
fn update_mtime(path: &Path) {
400-
let mut max = None;
401-
if let Ok(entries) = path.parent().unwrap().join("deps").read_dir() {
402-
for entry in entries.map(|e| t!(e)) {
403-
if t!(entry.file_type()).is_file() {
404-
let meta = t!(entry.metadata());
405-
let time = FileTime::from_last_modification_time(&meta);
406-
max = cmp::max(max, Some(time));
407-
}
408-
}
409-
}
410-
411-
if !max.is_none() && max <= Some(mtime(path)) {
412-
return
398+
///
399+
/// Note that we only consider Rust libraries as that's what we're interested in
400+
/// propagating changes from. Files like executables are tracked elsewhere.
401+
fn update_mtime(build: &Build, path: &Path) {
402+
let entries = match path.parent().unwrap().join("deps").read_dir() {
403+
Ok(entries) => entries,
404+
Err(_) => return,
405+
};
406+
let files = entries.map(|e| t!(e)).filter(|e| t!(e.file_type()).is_file());
407+
let files = files.filter(|e| {
408+
let filename = e.file_name();
409+
let filename = filename.to_str().unwrap();
410+
filename.ends_with(".rlib") ||
411+
filename.ends_with(".lib") ||
412+
is_dylib(&filename)
413+
});
414+
let max = files.max_by_key(|entry| {
415+
let meta = t!(entry.metadata());
416+
FileTime::from_last_modification_time(&meta)
417+
});
418+
let max = match max {
419+
Some(max) => max,
420+
None => return,
421+
};
422+
423+
if mtime(&max.path()) > mtime(path) {
424+
build.verbose(&format!("updating {:?} as {:?} changed", path, max.path()));
425+
t!(File::create(path));
413426
}
414-
t!(File::create(path));
415427
}

src/bootstrap/dist.rs

+7-4
Original file line numberDiff line numberDiff line change
@@ -357,14 +357,12 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) {
357357
pub fn rust_src(build: &Build) {
358358
println!("Dist src");
359359

360-
let plain_name = format!("rustc-{}-src", package_vers(build));
361360
let name = format!("rust-src-{}", package_vers(build));
362361
let image = tmpdir(build).join(format!("{}-image", name));
363362
let _ = fs::remove_dir_all(&image);
364363

365364
let dst = image.join("lib/rustlib/src");
366365
let dst_src = dst.join("rust");
367-
let plain_dst_src = dst.join(&plain_name);
368366
t!(fs::create_dir_all(&dst_src));
369367

370368
// This is the set of root paths which will become part of the source package
@@ -444,7 +442,11 @@ pub fn rust_src(build: &Build) {
444442
build.run(&mut cmd);
445443

446444
// Rename directory, so that root folder of tarball has the correct name
447-
t!(fs::rename(&dst_src, &plain_dst_src));
445+
let plain_name = format!("rustc-{}-src", package_vers(build));
446+
let plain_dst_src = tmpdir(build).join(&plain_name);
447+
let _ = fs::remove_dir_all(&plain_dst_src);
448+
t!(fs::create_dir_all(&plain_dst_src));
449+
cp_r(&dst_src, &plain_dst_src);
448450

449451
// Create the version file
450452
write_file(&plain_dst_src.join("version"), build.version.as_bytes());
@@ -453,10 +455,11 @@ pub fn rust_src(build: &Build) {
453455
let mut cmd = Command::new("tar");
454456
cmd.arg("-czf").arg(sanitize_sh(&rust_src_location(build)))
455457
.arg(&plain_name)
456-
.current_dir(&dst);
458+
.current_dir(tmpdir(build));
457459
build.run(&mut cmd);
458460

459461
t!(fs::remove_dir_all(&image));
462+
t!(fs::remove_dir_all(&plain_dst_src));
460463
}
461464

462465
fn install(src: &Path, dstdir: &Path, perms: u32) {

src/bootstrap/flags.rs

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ pub enum Subcommand {
6767
},
6868
Clean,
6969
Dist {
70+
paths: Vec<PathBuf>,
7071
install: bool,
7172
},
7273
}
@@ -249,6 +250,7 @@ To learn more about a subcommand, run `./x.py <command> -h`
249250
opts.optflag("", "install", "run installer as well");
250251
m = parse(&opts);
251252
Subcommand::Dist {
253+
paths: remaining_as_path(&m),
252254
install: m.opt_present("install"),
253255
}
254256
}

src/bootstrap/step.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -615,7 +615,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
615615
.default(true)
616616
.only_host_build(true)
617617
.run(move |s| dist::analysis(build, &s.compiler(), s.target));
618-
rules.dist("install", "src")
618+
rules.dist("install", "path/to/nowhere")
619619
.dep(|s| s.name("default:dist"))
620620
.run(move |s| install::install(build, s.stage, s.target));
621621

@@ -932,11 +932,11 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd?
932932
Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]),
933933
Subcommand::Test { ref paths, test_args: _ } => (Kind::Test, &paths[..]),
934934
Subcommand::Bench { ref paths, test_args: _ } => (Kind::Bench, &paths[..]),
935-
Subcommand::Dist { install } => {
935+
Subcommand::Dist { ref paths, install } => {
936936
if install {
937937
return vec![self.sbuild.name("install")]
938938
} else {
939-
(Kind::Dist, &[][..])
939+
(Kind::Dist, &paths[..])
940940
}
941941
}
942942
Subcommand::Clean => panic!(),

src/ci/docker/arm-android/Dockerfile

+6-1
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,9 @@ ENV RUST_CONFIGURE_ARGS \
4848
--armv7-linux-androideabi-ndk=/android/ndk-arm-9 \
4949
--i686-linux-android-ndk=/android/ndk-x86-9 \
5050
--aarch64-linux-android-ndk=/android/ndk-aarch64
51-
ENV XPY_CHECK test --target arm-linux-androideabi
51+
52+
# Just a smoke test in dist to see if this works for now, we should expand this
53+
# to all the targets above eventually.
54+
ENV SCRIPT \
55+
python2.7 ../x.py test --target arm-linux-androideabi && \
56+
python2.7 ../x.py dist --target arm-linux-androideabi

src/ci/docker/arm-android/start-emulator.sh

-1
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,4 @@ set -ex
1313
ANDROID_EMULATOR_FORCE_32BIT=true \
1414
nohup nohup emulator @arm-18 -no-window -partition-size 2047 \
1515
0<&- &>/dev/null &
16-
adb wait-for-device
1716
exec "$@"

src/ci/docker/cross/Dockerfile

+6-1
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,12 @@ ENV TARGETS=$TARGETS,wasm32-unknown-emscripten
5656
ENV RUST_CONFIGURE_ARGS \
5757
--target=$TARGETS \
5858
--enable-rustbuild
59-
ENV RUST_CHECK_TARGET ""
59+
60+
# Just a smoke test in dist to see if this works for now, we should expand this
61+
# to all the targets above eventually.
62+
ENV SCRIPT \
63+
python2.7 ../x.py build && \
64+
python2.7 ../x.py dist --target wasm32-unknown-emscripten
6065

6166
ENV AR_s390x_unknown_linux_gnu=s390x-linux-gnu-ar \
6267
CC_s390x_unknown_linux_gnu=s390x-linux-gnu-gcc \

src/ci/docker/dist-arm-unknown-linux-gnueabi/Dockerfile

+4-4
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
2424
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
2525

2626
ENV RUST_CONFIGURE_ARGS --host=arm-unknown-linux-gnueabi
27-
ENV XPY_RUN \
28-
dist \
29-
--host arm-unknown-linux-gnueabi \
30-
--target arm-unknown-linux-gnueabi
27+
ENV SCRIPT \
28+
python2.7 ../x.py dist \
29+
--host arm-unknown-linux-gnueabi \
30+
--target arm-unknown-linux-gnueabi

src/ci/docker/dist-x86_64-unknown-freebsd/Dockerfile

+4-4
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ ENV \
3232
CXX_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-g++
3333

3434
ENV RUST_CONFIGURE_ARGS --host=x86_64-unknown-freebsd
35-
ENV XPY_RUN \
36-
dist \
37-
--host x86_64-unknown-freebsd \
38-
--target x86_64-unknown-freebsd
35+
ENV SCRIPT \
36+
python2.7 ../x.py dist \
37+
--host x86_64-unknown-freebsd \
38+
--target x86_64-unknown-freebsd

src/ci/docker/i686-gnu/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
2323
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
2424

2525
ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
26-
ENV RUST_CHECK_TARGET check
26+
ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist

0 commit comments

Comments
 (0)