rustbuild: Tweak for vendored dependencies

A few changes are included here: * The `winapi` and `url` dependencies were dropped. The source code for these projects is pretty weighty, and we're about to vendor them, so let's not commit to that intake just yet. If necessary we can vendor them later but for now it shouldn't be necessary. * The `--frozen` flag is now always passed to Cargo, obviating the need for tidy's `cargo_lock` check. * Tidy was updated to not check the vendor directory Closes #34687
rust-lang · 31a8638e5e716bec90f4398a57c58fb34e492667 · alexcrichton committed Nov 1, 2016 · Nov 1, 2016 · 31a8638
1 parent 319f46f
commit 31a8638e5e716bec90f4398a57c58fb34e492667
diff --git a/.gitattributes b/.gitattributes
@@ -7,3 +7,4 @@
 src/etc/pkg/rust-logo.ico binary
 src/etc/pkg/rust-logo.png binary
 *.woff binary
+src/vendor/* binary
diff --git a/.gitignore b/.gitignore
@@ -98,3 +98,4 @@ tmp.*.rs
 version.md
 version.ml
 version.texi
+.cargo
diff --git a/.travis.yml b/.travis.yml
@@ -15,7 +15,7 @@ before_install:
 script:
   - docker run -v `pwd`:/build rust
     sh -c "
-      ./configure --enable-rustbuild --llvm-root=/usr/lib/llvm-3.7 --enable-quiet-tests &&
+      ./configure --enable-vendor --enable-rustbuild --llvm-root=/usr/lib/llvm-3.7 --enable-quiet-tests &&
       make tidy &&
       make check -j4
     "
diff --git a/configure b/configure
@@ -634,6 +634,7 @@ opt rustbuild 0 "use the rust and cargo based build system"
 opt codegen-tests 1 "run the src/test/codegen tests"
 opt option-checking 1 "complain about unrecognized options in this configure script"
 opt ninja 0 "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)"
+opt vendor 0 "enable usage of vendored Rust crates"
 
 # Optimization and debugging options. These may be overridden by the release channel, etc.
 opt_nosave optimize 1 "build optimized rust code"
diff --git a/mk/dist.mk b/mk/dist.mk
@@ -65,7 +65,8 @@ PKG_FILES := \
       stage0.txt                               \
       rust-installer                           \
       tools                                    \
-      test)                                    \
+      test                                     \
+      vendor)                                  \
     $(PKG_GITMODULES)                          \
     $(filter-out config.stamp, \
                  $(MKFILES_FOR_TARBALL))
diff --git a/src/Cargo.lock b/src/Cargo.lock
@@ -44,13 +44,11 @@ dependencies = [
  "filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)",
  "getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)",
- "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
  "md5 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -150,25 +148,6 @@ name = "graphviz"
 version = "0.0.0"
 
 [[package]]
-name = "idna"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "matches 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "unicode-bidi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "kernel32-sys"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
 name = "libc"
 version = "0.0.0"
 dependencies = [
@@ -183,9 +162,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 [[package]]
 name = "linkchecker"
 version = "0.1.0"
-dependencies = [
- "url 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
-]
 
 [[package]]
 name = "log"
@@ -197,11 +173,6 @@ version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
-name = "matches"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
 name = "md5"
 version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -705,55 +676,15 @@ dependencies = [
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "unicode-bidi"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "matches 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "unicode-normalization"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "url"
-version = "1.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "matches 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "winapi"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "winapi-build"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [metadata]
 "checksum cmake 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "0e5bcf27e097a184c1df4437654ed98df3d7a516e8508a6ba45d8b092bbdf283"
 "checksum env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "15abd780e45b3ea4f76b4e9a26ff4843258dd8a3eed2775a0e7368c2e7936c2f"
 "checksum filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "5363ab8e4139b8568a6237db5248646e5a8a2f89bd5ccb02092182b11fd3e922"
 "checksum gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)" = "553f11439bdefe755bf366b264820f1da70f3aaf3924e594b886beb9c831bcf5"
 "checksum getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9047cfbd08a437050b363d35ef160452c5fe8ea5187ae0a624708c91581d685"
-"checksum idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1053236e00ce4f668aeca4a769a09b3bf5a682d802abd6f3cb39374f6b162c11"
-"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
 "checksum libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "044d1360593a78f5c8e5e710beccdc24ab71d1f01bc19a29bcacdba22e8475d8"
 "checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054"
-"checksum matches 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bcc3ad8109fa4b522f9b0cd81440422781f564aaf8c195de6b9d6642177ad0dd"
 "checksum md5 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a5539a8dee9b4ae308c9c406a379838b435a8f2c84cf9fedc6d5a576be9888db"
 "checksum num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "cee7e88156f3f9e19bdd598f8d6c9db7bf4078f99f8381f43a55b09648d1a6e3"
 "checksum rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)" = "6159e4e6e559c81bd706afe9c8fd68f547d3e851ce12e76b1de7914bab61691b"
 "checksum toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)" = "0590d72182e50e879c4da3b11c6488dae18fccb1ae0c7a3eda18e16795844796"
-"checksum unicode-bidi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c1f7ceb96afdfeedee42bade65a0d585a6a0106f681b6749c8ff4daa8df30b3f"
-"checksum unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "26643a2f83bac55f1976fb716c10234485f9202dcd65cfbdf9da49867b271172"
-"checksum url 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "48ccf7bd87a81b769cf84ad556e034541fb90e1cd6d4bc375c822ed9500cd9d7"
-"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
-"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml
@@ -27,10 +27,6 @@ num_cpus = "0.2"
 toml = "0.1"
 getopts = "0.2"
 rustc-serialize = "0.3"
-gcc = "0.3.36"
+gcc = "0.3.38"
 libc = "0.2"
 md5 = "0.1"
-
-[target.'cfg(windows)'.dependencies]
-winapi = "0.2"
-kernel32-sys = "0.2"
diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py
@@ -259,9 +259,11 @@ def build_bootstrap(self):
         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
                       os.pathsep + env["PATH"]
-        self.run([self.cargo(), "build", "--manifest-path",
-                  os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")],
-                 env)
+        args = [self.cargo(), "build", "--manifest-path",
+                os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
+        if self.use_vendored_sources:
+            args.append("--frozen")
+        self.run(args, env)
 
     def run(self, args, env):
         proc = subprocess.Popen(args, env=env)
@@ -384,6 +386,25 @@ def main():
     except:
         pass
 
+    rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
+                              'CFG_ENABLE_VENDOR' in rb.config_mk
+
+    if rb.use_vendored_sources:
+        if not os.path.exists('.cargo'):
+            os.makedirs('.cargo')
+        f = open('.cargo/config','w')
+        f.write("""
+            [source.crates-io]
+            replace-with = 'vendored-sources'
+            registry = 'https://example.com'
+
+            [source.vendored-sources]
+            directory = '{}/src/vendor'
+        """.format(rb.rust_root))
+        f.close()
+    else:
+        if os.path.exists('.cargo'):
+            shutil.rmtree('.cargo')
     data = stage0_data(rb.rust_root)
     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
     rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
@@ -44,6 +44,7 @@ pub struct Config {
     pub submodules: bool,
     pub compiler_docs: bool,
     pub docs: bool,
+    pub vendor: bool,
     pub target_config: HashMap<String, Target>,
 
     // llvm codegen options
@@ -126,6 +127,7 @@ struct Build {
     docs: Option<bool>,
     submodules: Option<bool>,
     gdb: Option<String>,
+    vendor: Option<bool>,
 }
 
 /// TOML representation of how the LLVM build is configured.
@@ -234,6 +236,7 @@ impl Config {
         set(&mut config.compiler_docs, build.compiler_docs);
         set(&mut config.docs, build.docs);
         set(&mut config.submodules, build.submodules);
+        set(&mut config.vendor, build.vendor);
 
         if let Some(ref llvm) = toml.llvm {
             set(&mut config.ccache, llvm.ccache);
@@ -347,6 +350,7 @@ impl Config {
                 ("LOCAL_REBUILD", self.local_rebuild),
                 ("NINJA", self.ninja),
                 ("CODEGEN_TESTS", self.codegen_tests),
+                ("VENDOR", self.vendor),
             }
 
             match key {
diff --git a/src/bootstrap/config.toml.example b/src/bootstrap/config.toml.example
@@ -82,6 +82,9 @@
 # The path to (or name of) the GDB executable to use
 #gdb = "gdb"
 
+# Indicate whether the vendored sources are used for Rust dependencies or not
+#vendor = false
+
 # =============================================================================
 # Options for compiling Rust code itself
 # =============================================================================
diff --git a/src/bootstrap/job.rs b/src/bootstrap/job.rs
@@ -37,15 +37,82 @@
 //! Note that this module has a #[cfg(windows)] above it as none of this logic
 //! is required on Unix.
 
-extern crate kernel32;
-extern crate winapi;
+#![allow(bad_style, dead_code)]
 
 use std::env;
 use std::io;
 use std::mem;
 
-use self::winapi::*;
-use self::kernel32::*;
+type HANDLE = *mut u8;
+type BOOL = i32;
+type DWORD = u32;
+type LPHANDLE = *mut HANDLE;
+type LPVOID = *mut u8;
+type JOBOBJECTINFOCLASS = i32;
+type SIZE_T = usize;
+type LARGE_INTEGER = i64;
+type ULONG_PTR = usize;
+type ULONGLONG = u64;
+
+const FALSE: BOOL = 0;
+const DUPLICATE_SAME_ACCESS: DWORD = 0x2;
+const PROCESS_DUP_HANDLE: DWORD = 0x40;
+const JobObjectExtendedLimitInformation: JOBOBJECTINFOCLASS = 9;
+const JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE: DWORD = 0x2000;
+
+extern "system" {
+    fn CreateJobObjectW(lpJobAttributes: *mut u8, lpName: *const u8) -> HANDLE;
+    fn CloseHandle(hObject: HANDLE) -> BOOL;
+    fn GetCurrentProcess() -> HANDLE;
+    fn OpenProcess(dwDesiredAccess: DWORD,
+                   bInheritHandle: BOOL,
+                   dwProcessId: DWORD) -> HANDLE;
+    fn DuplicateHandle(hSourceProcessHandle: HANDLE,
+                       hSourceHandle: HANDLE,
+                       hTargetProcessHandle: HANDLE,
+                       lpTargetHandle: LPHANDLE,
+                       dwDesiredAccess: DWORD,
+                       bInheritHandle: BOOL,
+                       dwOptions: DWORD) -> BOOL;
+    fn AssignProcessToJobObject(hJob: HANDLE, hProcess: HANDLE) -> BOOL;
+    fn SetInformationJobObject(hJob: HANDLE,
+                               JobObjectInformationClass: JOBOBJECTINFOCLASS,
+                               lpJobObjectInformation: LPVOID,
+                               cbJobObjectInformationLength: DWORD) -> BOOL;
+}
+
+#[repr(C)]
+struct JOBOBJECT_EXTENDED_LIMIT_INFORMATION {
+    BasicLimitInformation: JOBOBJECT_BASIC_LIMIT_INFORMATION,
+    IoInfo: IO_COUNTERS,
+    ProcessMemoryLimit: SIZE_T,
+    JobMemoryLimit: SIZE_T,
+    PeakProcessMemoryUsed: SIZE_T,
+    PeakJobMemoryUsed: SIZE_T,
+}
+
+#[repr(C)]
+struct IO_COUNTERS {
+    ReadOperationCount: ULONGLONG,
+    WriteOperationCount: ULONGLONG,
+    OtherOperationCount: ULONGLONG,
+    ReadTransferCount: ULONGLONG,
+    WriteTransferCount: ULONGLONG,
+    OtherTransferCount: ULONGLONG,
+}
+
+#[repr(C)]
+struct JOBOBJECT_BASIC_LIMIT_INFORMATION {
+    PerProcessUserTimeLimit: LARGE_INTEGER,
+    PerJobUserTimeLimit: LARGE_INTEGER,
+    LimitFlags: DWORD,
+    MinimumWorkingsetSize: SIZE_T,
+    MaximumWorkingsetSize: SIZE_T,
+    ActiveProcessLimit: DWORD,
+    Affinity: ULONG_PTR,
+    PriorityClass: DWORD,
+    SchedulingClass: DWORD,
+}
 
 pub unsafe fn setup() {
     // Create a new job object for us to use
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
@@ -460,6 +460,9 @@ impl Build {
         if self.config.rust_optimize {
             cargo.arg("--release");
         }
+        if self.config.vendor {
+            cargo.arg("--frozen");
+        }
         return cargo
     }
 
diff --git a/src/tools/linkchecker/Cargo.toml b/src/tools/linkchecker/Cargo.toml
@@ -3,9 +3,6 @@ name = "linkchecker"
 version = "0.1.0"
 authors = ["Alex Crichton <[email protected]>"]
 
-[dependencies]
-url = "1.2"
-
 [[bin]]
 name = "linkchecker"
 path = "main.rs"