auto merge of #12793 : brson/rust/installer, r=alexcrichton

Work towards #9876.

Several minor things here:
  * Fix the `need_ok` function in `configure`
  * Install man pages with non-executable permissions
  * Use the correct directory for man pages when installing (this was a recent regression)
  * Put all distributables in a new `dist/` directory in the build directory (there are soon to be significantly more of these)

Finally, this also creates a new, more precise way to install and uninstall Rust's files, the `install.sh` script, and creates a build target (currently `dist-tar-bins`) that creates a binary tarball containing all the installable files, boilerplate and license docs, and `install.sh`.

This binary tarball is the lowest-common denominator way to install Rust on Unix. We'll use it as the default installer on Linux (OS X will use .pkg).

## How `install.sh` works

* First, the makefiles (`prepare.mk` and `dist.mk`) put all the stuff that needs to be installed in a new directory in `dist/`.
* Then it puts `install.sh` in that same directory and a list of all the files to install at `rustlib/manifest`.
* Then the directory can be packaged and distributed.
* When `install.sh` runs it does some sanity checking then copies everything in the manifest to the install prefix, then copies the manifest as well.
* When `install.sh` runs again in the future it first looks for the existing manifest at the install prefix, and if it exists deletes everything in it. This is how the core distribution is upgraded - cargo is responsible for the rest.
* `install.sh --uninstall` will uninstall Rust

## Future work:

  * Modify `install.sh` to accept `--man-dir` etc
  * Rewrite `install.mk` to delegate to `install.sh`
  * Investigate how `install.sh` does or doesn't work with .pkg on Mac
  * Modify `dist.mk` to create `.pkg` files for all hosts
  * Possibly use [makeself](http://www.megastep.org/makeself/) to create self-extracting installers
  * Modify dist-snap bots run on mac as well, uploading binary tarballs and .pkg files for the four combos of linux, mac, x86, and x86_64.
  * Adjust build system to be able to augment versions with '-nightly'
  * Adjust build system to name dist artifacts without version numbers e.g. `rust-nightly-...pkg`. This is so we don't leave a huge trail of old nightly binaries on S3 - they just get overwritten.
  * Create new dist-nightly builder
  * Give the build master a new cron job to push to dist-nightly every night
  * Add docs to distributables
  * Update README.md to reflect the new reality
  * Modernize the website to promote new installers

Author: bors

.gitignore

@@ -65,6 +65,7 @@ config.mk
 /mingw-build/
 src/.DS_Store
 /tmp/
+/dist/
 /stage0/
 /dl/
 /stage1/

configure

@@ -22,7 +22,7 @@ err() {
 need_ok() {
     if [ $? -ne 0 ]
     then
-        err $1
+        err "$1"
     fi
 }
 
@@ -340,7 +340,7 @@ DEFAULT_BUILD="${CFG_CPUTYPE}-${CFG_OSTYPE}"
 
 CFG_SRC_DIR="$(cd $(dirname $0) && pwd)/"
 CFG_BUILD_DIR="$(pwd)/"
-CFG_SELF=${CFG_SRC_DIR}$(basename $0)
+CFG_SELF="$0"
 CFG_CONFIGURE_ARGS="$@"
 
 OPTIONS=""
@@ -412,16 +412,15 @@ fi
 valopt libdir "${CFG_PREFIX}/${CFG_LIBDIR_RELATIVE}" "install libraries"
 valopt rustlibdir "rustlib" "subdirectory name for rustc's libraries"
 
-# Validate Options
-step_msg "validating $CFG_SELF args"
-validate_opt
-
 if [ $HELP -eq 1 ]
 then
     echo
     exit 0
 fi
 
+# Validate Options
+step_msg "validating $CFG_SELF args"
+validate_opt
 
 step_msg "looking for build programs"
 
@@ -728,7 +727,7 @@ step_msg "making directories"
 
 for i in \
     doc doc/std doc/extra \
-    dl tmp
+    dl tmp dist
 do
     make_dir $i
 done

mk/dist.mk

@@ -4,16 +4,16 @@
 
 PKG_NAME := rust
 PKG_DIR = $(PKG_NAME)-$(CFG_RELEASE)
-PKG_TAR = $(PKG_DIR).tar.gz
+PKG_TAR = dist/$(PKG_DIR).tar.gz
 
 ifdef CFG_ISCC
 PKG_ISS = $(wildcard $(S)src/etc/pkg/*.iss)
 PKG_ICO = $(S)src/etc/pkg/rust-logo.ico
-PKG_EXE = $(PKG_DIR)-install.exe
+PKG_EXE = dist/$(PKG_DIR)-install.exe
 endif
 
 ifeq ($(CFG_OSTYPE), apple-darwin)
-PKG_OSX = $(PKG_DIR).pkg
+PKG_OSX = dist/$(PKG_DIR).pkg
 endif
 
 PKG_GITMODULES := $(S)src/libuv $(S)src/llvm $(S)src/gyp $(S)src/compiler-rt
@@ -71,14 +71,15 @@ dist-prepare-win: PREPARE_DIR_CMD=$(DEFAULT_PREPARE_DIR_CMD)
 dist-prepare-win: PREPARE_BIN_CMD=$(DEFAULT_PREPARE_BIN_CMD)
 dist-prepare-win: PREPARE_LIB_CMD=$(DEFAULT_PREPARE_LIB_CMD)
 dist-prepare-win: PREPARE_MAN_CMD=$(DEFAULT_PREPARE_MAN_CMD)
+dist-prepare-win: PREPARE_CLEAN=true
 dist-prepare-win: prepare-base
 
 endif
 
 $(PKG_TAR): $(PKG_FILES)
 	@$(call E, making dist dir)
-	$(Q)rm -Rf dist
-	$(Q)mkdir -p dist/$(PKG_DIR)
+	$(Q)rm -Rf tmp/dist/$(PKG_DIR)
+	$(Q)mkdir -p tmp/dist/$(PKG_DIR)
 	$(Q)tar \
          -C $(S) \
          --exclude-vcs \
@@ -89,9 +90,9 @@ $(PKG_TAR): $(PKG_FILES)
          --exclude=*/llvm/test/*/*/*.ll \
          --exclude=*/llvm/test/*/*/*.td \
          --exclude=*/llvm/test/*/*/*.s \
-         -c $(UNROOTED_PKG_FILES) | tar -x -C dist/$(PKG_DIR)
-	$(Q)tar -czf $(PKG_TAR) -C dist $(PKG_DIR)
-	$(Q)rm -Rf dist
+         -c $(UNROOTED_PKG_FILES) | tar -x -C tmp/dist/$(PKG_DIR)
+	$(Q)tar -czf $(PKG_TAR) -C tmp/dist $(PKG_DIR)
+	$(Q)rm -Rf tmp/dist/$(PKG_DIR)
 
 .PHONY: dist distcheck
 
@@ -156,3 +157,35 @@ distcheck-osx: $(PKG_OSX)
 	@echo -----------------------------------------------
 
 endif
+
+dist-install-dir: $(foreach host,$(CFG_HOST),dist-install-dir-$(host))
+
+dist-tar-bins: $(foreach host,$(CFG_HOST),dist/$(PKG_DIR)-$(host).tar.gz)
+
+define DEF_INSTALLER
+dist-install-dir-$(1): PREPARE_HOST=$(1)
+dist-install-dir-$(1): PREPARE_TARGETS=$(1)
+dist-install-dir-$(1): PREPARE_STAGE=2
+dist-install-dir-$(1): PREPARE_DEST_DIR=tmp/dist/$$(PKG_DIR)-$(1)
+dist-install-dir-$(1): PREPARE_DIR_CMD=$(DEFAULT_PREPARE_DIR_CMD)
+dist-install-dir-$(1): PREPARE_BIN_CMD=$(DEFAULT_PREPARE_BIN_CMD)
+dist-install-dir-$(1): PREPARE_LIB_CMD=$(DEFAULT_PREPARE_LIB_CMD)
+dist-install-dir-$(1): PREPARE_MAN_CMD=$(DEFAULT_PREPARE_MAN_CMD)
+dist-install-dir-$(1): PREPARE_CLEAN=true
+dist-install-dir-$(1): prepare-base
+	$$(Q)(cd $$(PREPARE_DEST_DIR)/ && find -type f) \
+      > $$(PREPARE_DEST_DIR)/$$(CFG_LIBDIR_RELATIVE)/$$(CFG_RUSTLIBDIR)/manifest
+	$$(Q)$$(PREPARE_MAN_CMD) $$(S)COPYRIGHT $$(PREPARE_DEST_DIR)
+	$$(Q)$$(PREPARE_MAN_CMD) $$(S)LICENSE-APACHE $$(PREPARE_DEST_DIR)
+	$$(Q)$$(PREPARE_MAN_CMD) $$(S)LICENSE-MIT $$(PREPARE_DEST_DIR)
+	$$(Q)$$(PREPARE_MAN_CMD) $$(S)README.md $$(PREPARE_DEST_DIR)
+	$$(Q)$$(PREPARE_BIN_CMD) $$(S)src/etc/install.sh $$(PREPARE_DEST_DIR)
+
+dist/$$(PKG_DIR)-$(1).tar.gz: dist-install-dir-$(1)
+	@$(call E, build: $$@)
+	$$(Q)tar -czf dist/$$(PKG_DIR)-$(1).tar.gz -C tmp/dist $$(PKG_DIR)-$(1)
+
+endef
+
+$(foreach host,$(CFG_HOST),\
+  $(eval $(call DEF_INSTALLER,$(host))))

mk/prepare.mk

@@ -33,15 +33,15 @@ prepare-base: PREPARE_SOURCE_LIB_DIR=$(PREPARE_SOURCE_DIR)/$(CFG_LIBDIR_RELATIVE
 prepare-base: PREPARE_SOURCE_MAN_DIR=$(S)/man
 prepare-base: PREPARE_DEST_BIN_DIR=$(PREPARE_DEST_DIR)/bin
 prepare-base: PREPARE_DEST_LIB_DIR=$(PREPARE_DEST_DIR)/$(CFG_LIBDIR_RELATIVE)
-prepare-base: PREPARE_DEST_MAN_DIR=$(PREPARE_DEST_DIR)/man1
+prepare-base: PREPARE_DEST_MAN_DIR=$(PREPARE_DEST_DIR)/man/man1
 prepare-base: prepare-host prepare-targets
 
 prepare-everything: prepare-host prepare-targets
 
 DEFAULT_PREPARE_DIR_CMD = umask 022 && mkdir -p
 DEFAULT_PREPARE_BIN_CMD = install -m755
 DEFAULT_PREPARE_LIB_CMD = install -m644
-DEFAULT_PREPARE_MAN_CMD = install -m755
+DEFAULT_PREPARE_MAN_CMD = install -m644
 
 # On windows we install from stage3, but on unix only stage2
 # Because of the way these rules are organized, preparing from any
@@ -55,14 +55,14 @@ endif
 # Create a directory
 # $(1) is the directory
 define PREPARE_DIR
-	@$(Q)$(call E, install: $(1))
+	@$(Q)$(call E, prepare: $(1))
 	$(Q)$(PREPARE_DIR_CMD) $(1)
 endef
 
 # Copy an executable
 # $(1) is the filename/libname-glob
 define PREPARE_BIN
-	@$(call E, install: $(PREPARE_DEST_BIN_DIR)/$(1))
+	@$(call E, prepare: $(PREPARE_DEST_BIN_DIR)/$(1))
 	$(Q)$(PREPARE_BIN_CMD) $(PREPARE_SOURCE_BIN_DIR)/$(1) $(PREPARE_DEST_BIN_DIR)/$(1)
 endef
 
@@ -75,7 +75,7 @@ endef
 # problem. I'm sorry, just don't remove the $(nop), alright?
 define PREPARE_LIB
 	$(nop)
-	@$(call E, install: $(PREPARE_WORKING_DEST_LIB_DIR)/$(1))
+	@$(call E, prepare: $(PREPARE_WORKING_DEST_LIB_DIR)/$(1))
 	$(Q)LIB_NAME="$(notdir $(lastword $(wildcard $(PREPARE_WORKING_SOURCE_LIB_DIR)/$(1))))"; \
 	MATCHES="$(filter-out %$(notdir $(lastword $(wildcard $(PREPARE_WORKING_SOURCE_LIB_DIR)/$(1)))),\
                         $(wildcard $(PREPARE_WORKING_DEST_LIB_DIR)/$(1)))"; \
@@ -91,7 +91,7 @@ endef
 # Copy a man page
 # $(1) - source dir
 define PREPARE_MAN
-	@$(call E, install: $(PREPARE_DEST_MAN_DIR)/$(1))
+	@$(call E, prepare: $(PREPARE_DEST_MAN_DIR)/$(1))
 	$(Q)$(PREPARE_MAN_CMD) $(PREPARE_SOURCE_MAN_DIR)/$(1) $(PREPARE_DEST_MAN_DIR)/$(1)
 endef
 
@@ -106,7 +106,7 @@ prepare-host-tools: \
             $(foreach host,$(CFG_HOST),\
               prepare-host-tool-$(tool)-$(stage)-$(host))))
 
-prepare-host-dirs:
+prepare-host-dirs: prepare-maybe-clean
 	$(call PREPARE_DIR,$(PREPARE_DEST_BIN_DIR))
 	$(call PREPARE_DIR,$(PREPARE_DEST_LIB_DIR))
 	$(call PREPARE_DIR,$(PREPARE_DEST_MAN_DIR))
@@ -115,7 +115,8 @@ prepare-host-dirs:
 # $(2) is stage
 # $(3) is host
 define DEF_PREPARE_HOST_TOOL
-prepare-host-tool-$(1)-$(2)-$(3): $$(foreach dep,$$(TOOL_DEPS_$(1)),prepare-host-lib-$$(dep)-$(2)-$(3)) \
+prepare-host-tool-$(1)-$(2)-$(3): prepare-maybe-clean \
+                                  $$(foreach dep,$$(TOOL_DEPS_$(1)),prepare-host-lib-$$(dep)-$(2)-$(3)) \
                                   $$(HBIN$(2)_H_$(3))/$(1)$$(X_$(3)) \
                                   prepare-host-dirs
 	$$(if $$(findstring $(2), $$(PREPARE_STAGE)),\
@@ -140,7 +141,8 @@ $(foreach tool,$(PREPARE_TOOLS),\
 define DEF_PREPARE_HOST_LIB
 prepare-host-lib-$(1)-$(2)-$(3): PREPARE_WORKING_SOURCE_LIB_DIR=$$(PREPARE_SOURCE_LIB_DIR)
 prepare-host-lib-$(1)-$(2)-$(3): PREPARE_WORKING_DEST_LIB_DIR=$$(PREPARE_DEST_LIB_DIR)
-prepare-host-lib-$(1)-$(2)-$(3): $$(foreach dep,$$(RUST_DEPS_$(1)),prepare-host-lib-$$(dep)-$(2)-$(3))\
+prepare-host-lib-$(1)-$(2)-$(3): prepare-maybe-clean \
+                                 $$(foreach dep,$$(RUST_DEPS_$(1)),prepare-host-lib-$$(dep)-$(2)-$(3))\
                                  $$(HLIB$(2)_H_$(3))/stamp.$(1) \
                                  prepare-host-dirs
 	$$(if $$(findstring $(2), $$(PREPARE_STAGE)),\
@@ -166,7 +168,7 @@ define DEF_PREPARE_TARGET_N
 # Rebind PREPARE_*_LIB_DIR to point to rustlib, then install the libs for the targets
 prepare-target-$(2)-host-$(3)-$(1): PREPARE_WORKING_SOURCE_LIB_DIR=$$(PREPARE_SOURCE_LIB_DIR)/$$(CFG_RUSTLIBDIR)/$(2)/lib
 prepare-target-$(2)-host-$(3)-$(1): PREPARE_WORKING_DEST_LIB_DIR=$$(PREPARE_DEST_LIB_DIR)/$$(CFG_RUSTLIBDIR)/$(2)/lib
-prepare-target-$(2)-host-$(3)-$(1): \
+prepare-target-$(2)-host-$(3)-$(1): prepare-maybe-clean \
         $$(foreach crate,$$(TARGET_CRATES), \
           $$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$$(crate)) \
         $$(if $$(findstring $(2),$$(CFG_HOST)), \
@@ -194,3 +196,9 @@ $(foreach host,$(CFG_HOST),\
   $(foreach target,$(CFG_TARGET), \
     $(foreach stage,$(PREPARE_STAGES),\
       $(eval $(call DEF_PREPARE_TARGET_N,$(stage),$(target),$(host))))))
+
+prepare-maybe-clean:
+	$(if $(findstring true,$(PREPARE_CLEAN)),\
+      @$(call E, cleaning destination $@),)
+	$(if $(findstring true,$(PREPARE_CLEAN)),\
+      $(Q)rm -rf $(PREPARE_DEST_DIR),)