Revert "Migrate building SwiftSyntax to swift_build_support"

CMakeLists.txt

@@ -1096,6 +1096,12 @@ endif()
 # declares the swift-stdlib-* set of targets. These targets will then
 # implicitly depend on any targets declared with IS_STDLIB.
 #
+# One such library that declares IS_STDLIB is SwiftSyntax, living in
+# tools/SwiftSyntax. If we include stdlib/ after tools/,
+# the swift-stdlib-* set of targets will not have been generated yet,
+# causing the implicit dependency for SwiftSyntax to silently not be
+# created. This then will cause SwiftSyntax to fail to build.
+#
 # https://bugs.swift.org/browse/SR-5975
 if(SWIFT_BUILD_STDLIB)
   add_subdirectory(stdlib)
@@ -1120,7 +1126,7 @@ add_subdirectory(include)
 
 if(SWIFT_INCLUDE_TOOLS)
   add_subdirectory(lib)
-
+  
   # Always include this after including stdlib/!
   # Refer to the large comment above the add_subdirectory(stdlib) call.
   # https://bugs.swift.org/browse/SR-5975

benchmark/scripts/test_Benchmark_Driver.py

@@ -17,6 +17,7 @@
 import os
 import time
 import unittest
+
 from StringIO import StringIO
 from imp import load_source
 

benchmark/scripts/test_utils.py

@@ -24,6 +24,7 @@
 
 import logging
 import sys
+
 from StringIO import StringIO
 from contextlib import contextmanager
 

utils/build-script

@@ -338,6 +338,7 @@ class BuildScriptInvocation(object):
             "--libicu-build-type", args.libicu_build_variant,
             "--xctest-build-type", args.build_variant,
             "--swiftpm-build-type", args.build_variant,
+            "--swiftsyntax-build-type", args.build_variant,
             "--llbuild-build-type", args.build_variant,
             "--swift-enable-assertions", str(args.swift_assertions).lower(),
             "--swift-stdlib-enable-assertions", str(
@@ -466,6 +467,8 @@ class BuildScriptInvocation(object):
             impl_args += ["--skip-build-libicu"]
         if not args.build_swiftpm:
             impl_args += ["--skip-build-swiftpm"]
+        if not args.build_swiftsyntax:
+            impl_args += ["--skip-build-swiftsyntax"]
         if not args.build_playgroundsupport:
             impl_args += ["--skip-build-playgroundsupport"]
         if args.build_swift_dynamic_stdlib:
@@ -509,6 +512,7 @@ class BuildScriptInvocation(object):
                           "--skip-test-lldb",
                           "--skip-test-llbuild",
                           "--skip-test-swiftpm",
+                          "--skip-test-swiftsyntax",
                           "--skip-test-xctest",
                           "--skip-test-foundation",
                           "--skip-test-libdispatch",
@@ -815,14 +819,10 @@ class BuildScriptInvocation(object):
                     build_dir=self.workspace.build_dir(
                         host_target, product_name))
                 if product.should_build(host_target):
-                    print("--- Building %s ---" % product_name)
                     product.build(host_target)
                 if product.should_test(host_target):
-                    print("--- Running tests for %s ---" % product_name)
                     product.test(host_target)
-                    print("--- Finished tests for %s ---" % product_name)
                 if product.should_install(host_target):
-                    print("--- Installing %s ---" % product_name)
                     product.install(host_target)
 
         # Extract symbols...

utils/build-script-impl

@@ -79,6 +79,10 @@ KNOWN_SETTINGS=(
     playgroundsupport-build-type "Debug"         "the build variant for PlaygroundSupport"
     xctest-build-type           "Debug"          "the build variant for xctest"
     swiftpm-build-type          "Debug"          "the build variant for swiftpm"
+    swiftsyntax-build-type      "Debug"          "the build variant for swiftSyntax"
+    # When this flag is set, the build-script will only build/install the swift-syntax parser
+    # This is a temporary workaround of having a separate build product for swift-syntax parser
+    skip-swiftsyntax-swiftside  ""               "skip building/installing the swift side of swiftsyntax"
     llbuild-enable-assertions   "1"              "enable assertions in llbuild"
     enable-asan                 ""               "enable Address Sanitizer"
     enable-ubsan                ""               "enable Undefined Behavior Sanitizer"
@@ -97,6 +101,8 @@ KNOWN_SETTINGS=(
     installable-package         ""               "the path to the archive of the installation directory"
     test-installable-package    ""               "whether to run post-packaging tests on the produced package"
     reconfigure                 ""               "force a CMake configuration run even if CMakeCache.txt already exists"
+    build-libparser-only        ""               "only build libSwiftSyntaxParser"
+    libparser-ver               ""               "current version of libSwiftSyntaxParser"
     skip-reconfigure            ""               "set to skip reconfigure"
     swift-primary-variant-sdk   ""               "default SDK for target binaries"
     swift-primary-variant-arch  ""               "default arch for target binaries"
@@ -120,6 +126,7 @@ KNOWN_SETTINGS=(
     skip-build-llbuild          ""               "set to skip building llbuild"
     skip-build-libcxx           ""               "set to skip building libcxx"
     skip-build-swiftpm          ""               "set to skip building swiftpm"
+    skip-build-swiftsyntax      ""               "set to skip building swiftSyntax"
     skip-build-xctest           ""               "set to skip building xctest"
     skip-build-foundation       ""               "set to skip building foundation"
     skip-build-libdispatch      ""               "set to skip building libdispatch"
@@ -133,6 +140,7 @@ KNOWN_SETTINGS=(
     skip-test-swift             ""               "set to skip testing Swift"
     skip-test-llbuild           ""               "set to skip testing llbuild"
     skip-test-swiftpm           ""               "set to skip testing swiftpm"
+    skip-test-swiftsyntax       ""               "set to skip testing swiftSyntax"
     skip-test-xctest            ""               "set to skip testing xctest"
     skip-test-foundation        ""               "set to skip testing foundation"
     skip-test-libdispatch       ""               "set to skip testing libdispatch"
@@ -190,6 +198,9 @@ KNOWN_SETTINGS=(
     install-lldb                ""               "whether to install LLDB"
     install-llbuild             ""               "whether to install llbuild"
     install-swiftpm             ""               "whether to install swiftpm"
+    install-swiftsyntax         ""               "whether to install swiftsyntax"
+    skip-install-swiftsyntax-module ""           "set to skip installing swiftsyntax modules"
+    swiftsyntax-verify-generated-files ""        "set to verify that the generated files in the source tree match the ones that would be generated from current master"
     install-xctest              ""               "whether to install xctest"
     install-foundation          ""               "whether to install foundation"
     install-libcxx              ""               "whether to install libc++"
@@ -1180,6 +1191,7 @@ CMARK_SOURCE_DIR="${WORKSPACE}/cmark"
 LLDB_SOURCE_DIR="${WORKSPACE}/lldb"
 LLBUILD_SOURCE_DIR="${WORKSPACE}/llbuild"
 SWIFTPM_SOURCE_DIR="${WORKSPACE}/swiftpm"
+SWIFTSYNTAX_SOURCE_DIR="${WORKSPACE}/swift-syntax"
 STRESSTEST_PACKAGE_DIR="${WORKSPACE}/swift-stress-tester"
 XCTEST_SOURCE_DIR="${WORKSPACE}/swift-corelibs-xctest"
 FOUNDATION_SOURCE_DIR="${WORKSPACE}/swift-corelibs-foundation"
@@ -1233,8 +1245,8 @@ PRODUCTS=("${PRODUCTS[@]}" swift)
 if [[ ! "${SKIP_BUILD_LLDB}" ]] ; then
      PRODUCTS=("${PRODUCTS[@]}" lldb)
 fi
-# LLBuild, SwiftPM and XCTest are dependent on Foundation, so Foundation must
-# be added to the list of build products first.
+# LLBuild, SwiftPM, SwiftSyntax and XCTest are dependent on Foundation, so
+# Foundation must be added to the list of build products first.
 if [[ ! "${SKIP_BUILD_LIBDISPATCH}" ]] ; then
      PRODUCTS=("${PRODUCTS[@]}" libdispatch)
      if [[ -z "${SKIP_BUILD_SWIFT_STATIC_LIBDISPATCH}" ]] ; then
@@ -1253,14 +1265,19 @@ fi
 if [[ ! "${SKIP_BUILD_PLAYGROUNDSUPPORT}" ]] ; then
      PRODUCTS=("${PRODUCTS[@]}" playgroundsupport)
 fi
-# SwiftPM is dependent on XCTest, so XCTest must be added to the list of build
-# products first.
+# SwiftPM and SwiftSyntax are dependent on XCTest, so XCTest must be added to
+# the list of build products first.
 if [[ ! "${SKIP_BUILD_XCTEST}" ]] ; then
      PRODUCTS=("${PRODUCTS[@]}" xctest)
 fi
+# SwiftSyntax is dependent on SwiftPM, so SwiftPM must be added to the list of
+# build products first.
 if [[ ! "${SKIP_BUILD_SWIFTPM}" ]] ; then
      PRODUCTS=("${PRODUCTS[@]}" swiftpm)
 fi
+if [[ ! "${SKIP_BUILD_SWIFTSYNTAX}" ]] ; then
+     PRODUCTS=("${PRODUCTS[@]}" swiftsyntax)
+fi
 
 # Checks if a given product is enabled (i.e. part of $PRODUCTS array)
 function contains_product() {
@@ -1566,6 +1583,9 @@ function build_directory_bin() {
             swiftpm)
                 echo "${root}/${SWIFTPM_BUILD_TYPE}/bin"
                 ;;
+            swiftsyntax)
+                echo "${root}/${SWIFTSYNTAX_BUILD_TYPE}/bin"
+                ;;
             xctest)
                 echo "${root}/${XCTEST_BUILD_TYPE}/bin"
                 ;;
@@ -1708,6 +1728,9 @@ function cmake_config_opt() {
             swiftpm)
                 echo "--config ${SWIFTPM_BUILD_TYPE}"
                 ;;
+            swiftsyntax)
+                echo "--config ${SWIFTSYNTAX_BUILD_TYPE}"
+                ;;
             xctest)
                 echo "--config ${XCTEST_BUILD_TYPE}"
                 ;;
@@ -1798,13 +1821,43 @@ function set_swiftpm_bootstrap_command() {
 
 function swiftpm_find_tool() {
     tool=$1
-    if [[ "${SKIP_BUILD_SWIFTPM}" ]]; then
+    if [[ "${SKIP_BUILD_SWIFTPM}" || "${BUILD_LIBPARSER_ONLY}" ]]; then
         echo "$(xcrun_find_tool ${tool})"
     else
         echo "$(build_directory_bin ${LOCAL_HOST} swiftpm)/${tool}"
     fi
 }
 
+function set_swiftsyntax_build_command() {
+    if [ "${BUILD_LIBPARSER_ONLY}" ]; then
+        # we don't have a compiler built so we have to use the one in the environment.
+        SWIFTC_BIN="$(xcrun_find_tool swiftc)"
+    else
+        SWIFTC_BIN="$(build_directory_bin ${LOCAL_HOST} swift)/swiftc"
+    fi
+
+    swiftsyntax_build_command=("${SWIFTSYNTAX_SOURCE_DIR}/build-script.py")
+    # Add --release if we have to build in release mode.
+    if [[ $(is_cmake_release_build_type "${SWIFTSYNTAX_BUILD_TYPE}") ]] ; then
+        swiftsyntax_build_command+=(--release)
+    fi
+    if [[ "${VERBOSE_BUILD}" ]] ; then
+        swiftsyntax_build_command+=(-v)
+    fi
+    swiftsyntax_build_command+=(
+        --build-dir="$(build_directory ${host} swiftsyntax)"
+        --swift-build-exec="$(swiftpm_find_tool swift-build)"
+        --swift-test-exec="$(swiftpm_find_tool swift-test)"
+        --swiftc-exec="${SWIFTC_BIN}"
+        --syntax-parser-header-dir="${SWIFT_SOURCE_DIR}/include/swift-c/SyntaxParser"
+        --syntax-parser-lib-dir="$(build_directory ${host} swift)/lib"
+        --swift-syntax-test-exec="$(build_directory_bin ${LOCAL_HOST} swift)/swift-syntax-test"
+        --filecheck-exec="$(build_directory_bin ${LOCAL_HOST} llvm)/FileCheck")
+    if [[ "${SWIFTSYNTAX_VERIFY_GENERATED_FILES}" ]] ; then
+        swiftsyntax_build_command+=(--verify-generated-files)
+    fi
+}
+
 #
 # Configure and build each product
 #
@@ -2305,6 +2358,15 @@ for host in "${ALL_HOSTS[@]}"; do
                     build_targets=("${build_targets[@]}"
                                    "${SWIFT_BENCHMARK_TARGETS[@]}")
                 fi
+                if [ "${BUILD_LIBPARSER_ONLY}" ]; then
+                    build_targets=(libSwiftSyntaxParser)
+                    if [ "${LIBPARSER_VER}" ] ; then
+                      cmake_options=(
+                          "${cmake_options[@]}"
+                          -DSWIFT_LIBPARSER_VER:STRING="${LIBPARSER_VER}"
+                      )
+                    fi
+                fi
                 skip_build=${SKIP_BUILD_SWIFT}
                 ;;
             lldb)
@@ -2441,6 +2503,15 @@ for host in "${ALL_HOSTS[@]}"; do
                 call "${swiftpm_bootstrap_command[@]}"
 
                 # swiftpm installs itself with a bootstrap method. No further cmake building is performed.
+                continue
+                ;;
+            swiftsyntax)
+                if [[ "${SKIP_SWIFTSYNTAX_SWIFTSIDE}" ]]; then
+                    continue
+                fi
+                set_swiftsyntax_build_command
+                call "${swiftsyntax_build_command[@]}"
+
                 continue
                 ;;
             xctest)
@@ -2945,6 +3016,16 @@ for host in "${ALL_HOSTS[@]}"; do
                 # As swiftpm tests itself, we break early here.
                 continue
                 ;;
+            swiftsyntax)
+                if [[ "${SKIP_TEST_SWIFTSYNTAX}" ]]; then
+                    continue
+                fi
+                echo "--- Running tests for ${product} ---"
+                set_swiftsyntax_build_command
+                call "${swiftsyntax_build_command[@]}" -t
+                # As swiftSyntax tests itself, we break early here.
+                continue
+                ;;
             xctest)
                 if [[ "${SKIP_TEST_XCTEST}" ]]; then
                     continue
@@ -3229,6 +3310,11 @@ for host in "${ALL_HOSTS[@]}"; do
                     continue
                 fi
                 INSTALL_TARGETS=install-swift-components
+                # Swift syntax parser is currently a sub-product of Swift;
+                # We need to specify the install target separately here.
+                if [ "${BUILD_LIBPARSER_ONLY}" ]; then
+                    INSTALL_TARGETS=tools/libSwiftSyntaxParser/install
+                fi
                 ;;
             llbuild)
                 if [[ -z "${INSTALL_LLBUILD}" ]] ; then
@@ -3259,6 +3345,43 @@ for host in "${ALL_HOSTS[@]}"; do
                 set_swiftpm_bootstrap_command
                 call "${swiftpm_bootstrap_command[@]}" --prefix="${host_install_destdir}${host_install_prefix}" install
                 # As swiftpm bootstraps the installation itself, we break early here.
+                continue
+                ;;
+            swiftsyntax)
+                if [[ -z "${INSTALL_SWIFTSYNTAX}" ]] ; then
+                    continue
+                fi
+                if [[ -z "${INSTALL_DESTDIR}" ]] ; then
+                    echo "--install-destdir is required to install products."
+                    exit 1
+                fi
+                echo "--- Installing ${product} ---"
+                if [ "${BUILD_LIBPARSER_ONLY}" ]; then
+                    # We don't have a toolchain so we should install to the specified dir
+                    DYLIB_DIR="${INSTALL_DESTDIR}"
+                    MODULE_DIR="${INSTALL_DESTDIR}/${product}.swiftmodule"
+                    # Create the install dir if it doesn't exist
+                    call mkdir -p "${INSTALL_DESTDIR}"
+                    # Install libParser is necessary
+                    rsync -a "$(build_directory ${host} swift)/lib/lib_InternalSwiftSyntaxParser.dylib" "${INSTALL_DESTDIR}"
+                    # Install module map of libParser so client can import SwiftSyntax
+                    rsync -a "${SWIFT_SOURCE_DIR}/include/swift-c/SyntaxParser" "${INSTALL_DESTDIR}"
+                else
+                    # We have a toolchain so install to the toolchain
+                    DYLIB_DIR="${host_install_destdir}${host_install_prefix}/lib/swift/${SWIFT_HOST_VARIANT}"
+                    MODULE_DIR="${DYLIB_DIR}/${product}.swiftmodule"
+                fi
+                if [[ "${SKIP_SWIFTSYNTAX_SWIFTSIDE}" ]]; then
+                    continue
+                fi
+                set_swiftsyntax_build_command
+                if [[ -z "${SKIP_INSTALL_SWIFTSYNTAX_MODULE}" ]] ; then
+                    mkdir -p "${MODULE_DIR}"
+                    call "${swiftsyntax_build_command[@]}" --dylib-dir="${DYLIB_DIR}" --swiftmodule-base-name "${MODULE_DIR}/${SWIFT_HOST_VARIANT_ARCH}" --install
+                else
+                    call "${swiftsyntax_build_command[@]}" --dylib-dir="${DYLIB_DIR}" --install
+                fi
+
                 continue
                 ;;
             xctest)

utils/build_swift/driver_arguments.py

@@ -203,7 +203,6 @@ def _apply_default_arguments(args):
         args.test_tvos = False
         args.test_watchos = False
         args.test_android = False
-        args.test_swiftsyntax = False
         args.test_indexstoredb = False
         args.test_sourcekitlsp = False
         args.test_skstresstester = False
@@ -569,15 +568,6 @@ def create_argument_parser():
            help='build IndexStoreDB')
     option(['--sourcekit-lsp'], toggle_true('build_sourcekitlsp'),
            help='build SourceKitLSP')
-    option('--install-swiftsyntax', toggle_true('install_swiftsyntax'),
-           help='install SwiftSyntax')
-    option('--skip-install-swiftsyntax-module',
-           toggle_true('skip_install_swiftsyntax_module'),
-           help='skip installing the SwiftSyntax modules')
-    option('--swiftsyntax-verify-generated-files',
-           toggle_true('swiftsyntax_verify_generated_files'),
-           help='set to verify that the generated files in the source tree '
-                'match the ones that would be generated from current master')
     option(['--install-sourcekit-lsp'], toggle_true('install_sourcekitlsp'),
            help='install SourceKitLSP')
     option(['--install-skstresstester'], toggle_true('install_skstresstester'),
@@ -967,8 +957,6 @@ def create_argument_parser():
            help='skip testing Android device targets on the host machine (the '
                 'phone itself)')
 
-    option('--skip-test-swiftsyntax', toggle_false('test_swiftsyntax'),
-           help='skip testing SwiftSyntax')
     option('--skip-test-indexstore-db', toggle_false('test_indexstoredb'),
            help='skip testing indexstore-db')
     option('--skip-test-sourcekit-lsp', toggle_false('test_sourcekitlsp'),