Merge branch 'main' into pr_model_improve

Author: zingo

shim/.buckconfig renamed to .Package.swift/backend_coreml/dummy.swift

@@ -8,15 +8,18 @@
   root = .
   prelude = third-party/prelude
   shim = shim
+  shim_et = shim_et
 
 [repository_aliases]
+  bazel_skylib = shim
   config = prelude
   ovr_config = prelude
-  toolchains = shim
-  fbcode = shim
+  toolchains = shim_et
+  fbcode = shim_et
   fbcode_macros = shim
-  fbsource = shim
+  fbsource = shim_et
   buck = shim
+  gh_facebook_buck2_shims_meta = shim
 
 [cxx]
   cxxflags = -g -std=c++17

shim/TARGETS renamed to .Package.swift/backend_coreml_debug/dummy.swift

@@ -1 +1 @@
-27e35de6c288bffad1b4d18b393579c1d1a95547
+08434df1f2f88c9770e59246caa2ff9c6f613270

.Package.swift/backend_mps/dummy.swift

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+apt-get update
+
+apt-get install -y --no-install-recommends openjdk-17-jdk

.Package.swift/backend_mps_debug/dummy.swift

@@ -30,6 +30,10 @@ ARG BUCK2_VERSION
 COPY ./common/install_buck.sh install_buck.sh
 RUN bash ./install_buck.sh && rm install_buck.sh
 
+# Install java
+COPY ./common/install_java.sh install_java.sh
+RUN bash ./install_java.sh && rm install_java.sh
+
 # Setup user
 COPY ./common/install_user.sh install_user.sh
 RUN bash ./install_user.sh && rm install_user.sh

.Package.swift/backend_xnnpack/dummy.swift

@@ -104,8 +104,12 @@ def model_should_run_on_target_os(model: str, target_os: str) -> bool:
     For example, a big model can be disabled in macos due to the limited macos resources.
     """
     if target_os == "macos":
+        # Disabled in macos due to limited resources, and should stay that way even if
+        # we otherwise re-enable.
         return model not in ["llava"]
-    return True
+    # Disabled globally because we have test-llava-runner-linux that does a more
+    # comprehensive E2E test of llava.
+    return model not in ["llava"]
 
 
 def export_models_for_ci() -> dict[str, dict]:

.Package.swift/backend_xnnpack_debug/dummy.swift

@@ -10,19 +10,17 @@ set -exu
 # shellcheck source=/dev/null
 source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
 
-BUILD_TOOL=$1
-if [[ -z "${BUILD_TOOL:-}" ]]; then
-  echo "Missing build tool (require buck2 or cmake), exiting..."
-  exit 1
-else
-  echo "Setup Linux for ${BUILD_TOOL} ..."
-fi
+read -r BUILD_TOOL BUILD_MODE EDITABLE < <(parse_args "$@")
 
 # As Linux job is running inside a Docker container, all of its dependencies
 # have already been installed, so we use PyTorch build from source here instead
 # of nightly. This allows CI to test against latest commits from PyTorch
-install_executorch "use-pt-pinned-commit"
-build_executorch_runner "${BUILD_TOOL}" "${2:-Release}"
+if [[ "${EDITABLE:-false}" == "true" ]]; then
+  install_executorch --use-pt-pinned-commit --editable
+else
+  install_executorch --use-pt-pinned-commit
+fi
+build_executorch_runner "${BUILD_TOOL}" "${BUILD_MODE}"
 
 if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then
   do_not_use_nightly_on_ci

.Package.swift/executorch/dummy.swift

@@ -10,13 +10,7 @@ set -exu
 # shellcheck source=/dev/null
 source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
 
-BUILD_TOOL=$1
-if [[ -z "${BUILD_TOOL:-}" ]]; then
-  echo "Missing build tool (require buck2 or cmake), exiting..."
-  exit 1
-else
-  echo "Setup MacOS for ${BUILD_TOOL} ..."
-fi
+read -r BUILD_TOOL BUILD_MODE EDITABLE < <(parse_args "$@")
 
 install_buck() {
   if ! command -v zstd &> /dev/null; then
@@ -135,8 +129,12 @@ print_cmake_info
 install_pytorch_and_domains
 # We build PyTorch from source here instead of using nightly. This allows CI to test against
 # the pinned commit from PyTorch
-install_executorch "use-pt-pinned-commit"
-build_executorch_runner "${BUILD_TOOL}" "${2:-Release}"
+if [[ "$EDITABLE" == "true" ]]; then
+  install_executorch --use-pt-pinned-commit --editable
+else
+  install_executorch --use-pt-pinned-commit
+fi
+build_executorch_runner "${BUILD_TOOL}" "${BUILD_MODE}"
 
 if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then
   do_not_use_nightly_on_ci

.Package.swift/executorch_debug/dummy.swift

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright (c) Qualcomm Innovation Center, Inc.
+# All rights reserved
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -exu
+
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
+export EXECUTORCH_ROOT="$(dirname "${BASH_SOURCE[0]}")/../.."
+
+if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
+  PYTHON_EXECUTABLE=python3
+fi
+
+which "${PYTHON_EXECUTABLE}"
+
+pushd $EXECUTORCH_ROOT/examples/apple/coreml/llama
+
+# Download stories llama110m artifacts
+download_stories_model_artifacts
+
+python export.py -n model.pte -p params.json -c stories110M.pt --seq_length 32 --max_seq_length 64 --dtype fp16 --coreml-quantize c4w
+
+popd

.Package.swift/kernels_custom/dummy.swift

@@ -91,7 +91,26 @@ test_model() {
     # Install requirements for llama vision.
     bash examples/models/llama3_2_vision/install_requirements.sh
   fi
-  # python3 -m examples.portable.scripts.export --model_name="llama2" should works too
+  if [[ "${MODEL_NAME}" == "qwen2_5" ]]; then
+      # Install requirements for export_llama
+      bash examples/models/llama/install_requirements.sh
+      # Test export_llama script: python3 -m examples.models.llama.export_llama.
+      # Use Llama random checkpoint with Qwen 2.5 1.5b model configuration.
+      "${PYTHON_EXECUTABLE}" -m examples.models.llama.export_llama --model "${MODEL_NAME}" -c examples/models/llama/params/demo_rand_params.pth -p examples/models/qwen2_5/1_5b_config.json
+      rm "./${MODEL_NAME}.pte"
+      return  # Skip running with portable executor runnner since portable doesn't support Qwen's biased linears.
+  fi
+  if [[ "${MODEL_NAME}" == "phi-4-mini" ]]; then
+      # Install requirements for export_llama
+      bash examples/models/llama/install_requirements.sh
+      # Test export_llama script: python3 -m examples.models.llama.export_llama.
+      "${PYTHON_EXECUTABLE}" -m examples.models.llama.export_llama --model "${MODEL_NAME}" -c examples/models/llama/params/demo_rand_params.pth -p examples/models/phi-4-mini/config.json
+      run_portable_executor_runner
+      rm "./${MODEL_NAME}.pte"
+      return
+  fi
+
+  # Export a basic .pte and run the model.
   "${PYTHON_EXECUTABLE}" -m examples.portable.scripts.export --model_name="${MODEL_NAME}" "${STRICT}"
   run_portable_executor_runner
 }
@@ -154,6 +173,7 @@ test_model_with_qnn() {
   export LD_LIBRARY_PATH=$QNN_SDK_ROOT/lib/x86_64-linux-clang/
   export PYTHONPATH=$EXECUTORCH_ROOT/..
 
+  EXTRA_FLAGS=""
   if [[ "${MODEL_NAME}" == "dl3" ]]; then
     EXPORT_SCRIPT=deeplab_v3
   elif [[ "${MODEL_NAME}" == "mv3" ]]; then
@@ -166,6 +186,12 @@ test_model_with_qnn() {
     EXPORT_SCRIPT=inception_v3
   elif [[ "${MODEL_NAME}" == "vit" ]]; then
     EXPORT_SCRIPT=torchvision_vit
+  elif [[ "${MODEL_NAME}" == "mb" ]]; then
+    EXPORT_SCRIPT=mobilebert_fine_tune
+    EXTRA_FLAGS="--num_epochs 1"
+    pip install scikit-learn
+  elif [[ "${MODEL_NAME}" == "w2l" ]]; then
+    EXPORT_SCRIPT=wav2letter
   elif [[ "${MODEL_NAME}" == "edsr" ]]; then
     EXPORT_SCRIPT=edsr
     # Additional deps for edsr
@@ -179,7 +205,7 @@ test_model_with_qnn() {
   # TODO(guangyang): Make QNN chipset matches the target device
   QNN_CHIPSET=SM8450
 
-  "${PYTHON_EXECUTABLE}" -m examples.qualcomm.scripts.${EXPORT_SCRIPT} -b ${CMAKE_OUTPUT_DIR} -m ${QNN_CHIPSET} --compile_only
+  "${PYTHON_EXECUTABLE}" -m examples.qualcomm.scripts.${EXPORT_SCRIPT} -b ${CMAKE_OUTPUT_DIR} -m ${QNN_CHIPSET} --compile_only $EXTRA_FLAGS
   EXPORTED_MODEL=$(find "./${EXPORT_SCRIPT}" -type f -name "${MODEL_NAME}*.pte" -print -quit)
 }
 

.Package.swift/kernels_custom_debug/dummy.swift

@@ -7,7 +7,11 @@
 set -eux
 
 # TODO: expand this to //...
-buck2 query //runtime/...
+# TODO: can't query cadence & vulkan backends
+buck2 query "//backends/apple/... + //backends/example/... + \
+//backends/mediatek/... + //backends/test/... + //backends/transforms/... + \
+//backends/xnnpack/... + //configurations/... + //kernels/portable/cpu/... + \
+//runtime/... + //schema/... + //test/... + //util/..."
 
 # TODO: expand the covered scope of Buck targets.
 buck2 build //runtime/core/portable_type/...

.Package.swift/kernels_optimized/dummy.swift

@@ -6,41 +6,36 @@
 # LICENSE file in the root directory of this source tree.
 set -eux
 
-BUILD_TOOL=$1
-if [[ $BUILD_TOOL =~ ^(cmake|buck2)$ ]]; then
-    echo "Running unittests for ${BUILD_TOOL} ..."
-else
-  echo "Missing build tool (require buck2 or cmake), exiting..."
-  exit 1
-fi
+# shellcheck source=/dev/null
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
 
-BUILD_MODE=$2
-if [[ "${BUILD_MODE:-}" =~ ^(Debug|Release)$ ]]; then
-    echo "Running tests in build mode ${BUILD_MODE} ..."
-else
-    echo "Unsupported build mode ${BUILD_MODE}, options are Debug or Release."
-    exit 1
-fi
+read -r BUILD_TOOL BUILD_MODE EDITABLE < <(parse_args "$@")
 
 # The generic Linux job chooses to use base env, not the one setup by the image
 eval "$(conda shell.bash hook)"
 CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
 conda activate "${CONDA_ENV}"
 
-# Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate
-source .ci/scripts/setup-vulkan-linux-deps.sh
+if [[ "$BUILD_TOOL" == "cmake" ]]; then
+    # Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate
+    source .ci/scripts/setup-vulkan-linux-deps.sh
 
-PYTHON_EXECUTABLE=python \
-EXECUTORCH_BUILD_PYBIND=ON \
-CMAKE_ARGS="-DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \
-.ci/scripts/setup-linux.sh "$BUILD_TOOL" "$BUILD_MODE"
+    PYTHON_EXECUTABLE=python \
+    EXECUTORCH_BUILD_PYBIND=ON \
+    CMAKE_ARGS="-DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \
+    .ci/scripts/setup-linux.sh "$@"
 
-# Install llama3_2_vision dependencies.
-PYTHON_EXECUTABLE=python ./examples/models/llama3_2_vision/install_requirements.sh
+    # Install llama3_2_vision dependencies.
+    PYTHON_EXECUTABLE=python ./examples/models/llama3_2_vision/install_requirements.sh
 
-if [[ "$BUILD_TOOL" == "cmake" ]]; then
     .ci/scripts/unittest-linux-cmake.sh
 elif [[ "$BUILD_TOOL" == "buck2" ]]; then
+    # Removing this breaks sccache in the Buck build, apparently
+    # because TMPDIR gets messed up? Please feel free to fix this and
+    # speed up this CI job!
+    PYTHON_EXECUTABLE=python \
+    .ci/scripts/setup-linux.sh "$@"
+
     .ci/scripts/unittest-buck2.sh
 else
     echo "Unknown build tool $BUILD_TOOL"

.Package.swift/kernels_optimized_debug/dummy.swift

@@ -6,21 +6,10 @@
 # LICENSE file in the root directory of this source tree.
 set -eux
 
-BUILD_TOOL=$1
-if [[ $BUILD_TOOL =~ ^(cmake|buck2)$ ]]; then
-    echo "Running unittests for ${BUILD_TOOL} ..."
-else
-  echo "Missing build tool (require buck2 or cmake), exiting..."
-  exit 1
-fi
+# shellcheck source=/dev/null
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
 
-BUILD_MODE=$2
-if [[ $BUILD_MODE =~ ^(Debug|Release)$ ]]; then
-    echo "Running tests in build mode ${BUILD_MODE} ..."
-else
-    echo "Unsupported build mode ${BUILD_MODE}, options are Debug or Release."
-    exit 1
-fi
+read -r BUILD_TOOL BUILD_MODE EDITABLE < <(parse_args "$@")
 
 bash .ci/scripts/setup-conda.sh
 eval "$(conda shell.bash hook)"
@@ -30,19 +19,19 @@ export TMP_DIR=$(mktemp -d)
 export PATH="${TMP_DIR}:$PATH"
 trap 'rm -rfv ${TMP_DIR}' EXIT
 
-# Setup MacOS dependencies as there is no Docker support on MacOS atm
-PYTHON_EXECUTABLE=python \
-EXECUTORCH_BUILD_PYBIND=ON \
-CMAKE_ARGS="-DEXECUTORCH_BUILD_COREML=ON -DEXECUTORCH_BUILD_MPS=ON -DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \
-${CONDA_RUN} --no-capture-output \
-.ci/scripts/setup-macos.sh "${BUILD_TOOL}" "${BUILD_MODE}"
+if [[ "$BUILD_TOOL" == "cmake" ]]; then
+    # Setup MacOS dependencies as there is no Docker support on MacOS atm
+    PYTHON_EXECUTABLE=python \
+    EXECUTORCH_BUILD_PYBIND=ON \
+    CMAKE_ARGS="-DEXECUTORCH_BUILD_COREML=ON -DEXECUTORCH_BUILD_MPS=ON -DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \
+    ${CONDA_RUN} --no-capture-output \
+    .ci/scripts/setup-macos.sh "$@"
 
-# Install llama3_2_vision dependencies.
-PYTHON_EXECUTABLE=python \
-${CONDA_RUN} --no-capture-output \
-./examples/models/llama3_2_vision/install_requirements.sh
+    # Install llama3_2_vision dependencies.
+    PYTHON_EXECUTABLE=python \
+    ${CONDA_RUN} --no-capture-output \
+    ./examples/models/llama3_2_vision/install_requirements.sh
 
-if [[ "$BUILD_TOOL" == "cmake" ]]; then
     .ci/scripts/unittest-macos-cmake.sh
 elif [[ "$BUILD_TOOL" == "buck2" ]]; then
     .ci/scripts/unittest-buck2.sh