Skip to content

Commit 5bad5a5

Browse files
zingozingo
authored
Merge branch 'main' into Arm-backend-Remove-unused-github-ci-git-setup
2 parents 7df3527 + 4236e5b commit 5bad5a5

File tree

921 files changed

+30291
-9144
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

921 files changed

+30291
-9144
lines changed

‎.ci/docker/ci_commit_pins/pytorch.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
7ae0ce6360b6e4f944906502d20da24c04debee5
1+
59d5cf083b4f860dea76fe8936076177f9367f10

‎.ci/docker/requirements-ci.txt

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ parameterized==0.9.0
1717

1818
# Doc build requirements, same as https://github.com/pytorch/pytorch/blob/main/.ci/docker/requirements-docs.txt
1919
sphinx==5.3.0
20+
sphinx-reredirects==0.1.4
2021
sphinx-gallery==0.14.0
2122
breathe==4.34.0
2223
exhale==0.2.3

‎.ci/scripts/build_android_instrumentation.sh

-21
This file was deleted.

‎.ci/scripts/check_c10_sync.sh

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -eux
9+
ls pytorch/.git || git clone https://github.com/pytorch/pytorch.git
10+
pytorch_pin="$(< .ci/docker/ci_commit_pins/pytorch.txt)"
11+
pushd pytorch
12+
git checkout "$pytorch_pin"
13+
popd
14+
"$(dirname "${BASH_SOURCE[0]}")"/compare_dirs.sh runtime/core/portable_type/c10/c10 pytorch/c10

‎.ci/scripts/compare_dirs.sh

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -eux
9+
10+
# Check if dir1's files are also found in dir2 with the same
11+
# contents. Exempt files named BUCK, CMakeLists.txt, TARGETS, or
12+
# targets.bzl.
13+
14+
if [ $# -ne 2 ]; then
15+
echo "Usage: $0 dir1 dir2" >&2
16+
exit 1
17+
fi
18+
dir1="$1"
19+
dir2="$2"
20+
21+
if [ ! -d "$dir1" ] || [ ! -d "$dir2" ]; then
22+
echo "Error: Both directories must exist" >&2
23+
exit 1
24+
fi
25+
26+
exit_status=0
27+
while IFS= read -r -d '' file; do
28+
base=$(basename "$file")
29+
case "$base" in
30+
"BUCK"|"CMakeLists.txt"|"TARGETS"|"targets.bzl")
31+
continue
32+
;;
33+
esac
34+
# Construct the corresponding path in the second directory
35+
file2="$dir2/${file#$dir1/}"
36+
# Check if the corresponding file exists in the second directory
37+
if [ ! -f "$file2" ]; then
38+
echo "Error: File '$file' found in '$dir1' but not found in '$dir2'" >&2
39+
exit 1
40+
fi
41+
# Compare the contents of the two files using diff
42+
set +ex
43+
differences=$(diff -u -p "$file" "$file2")
44+
set -e # leave x off
45+
# If there are any differences, print an error message and exit with failure status
46+
if [ -n "$differences" ]; then
47+
echo "Error: Mismatch detected in file '$file':" >&2
48+
echo "$differences" >&2
49+
exit_status=1
50+
fi
51+
set -x
52+
done < <(find "$dir1" -type f -print0)
53+
54+
exit $exit_status

‎.ci/scripts/gather_benchmark_configs.py

+2
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
"samsung_galaxy_s22": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/e59f866a-30aa-4aa1-87b7-4510e5820dfa",
2424
"samsung_galaxy_s24": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/98f8788c-2e25-4a3c-8bb2-0d1e8897c0db",
2525
"google_pixel_8_pro": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/d65096ab-900b-4521-be8b-a3619b69236a",
26+
"google_pixel_3_private_rooted": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/98d23ca8-ea9e-4fb7-b725-d402017b198d",
27+
"apple_iphone_15_private": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/55929353-2f28-4ee5-bdff-d1a95f58cb28",
2628
}
2729

2830
# Predefined benchmark configurations

‎.ci/scripts/test_ios_ci.sh

+5-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
set -e
99

10-
APP_PATH="examples/demo-apps/apple_ios/ExecuTorchDemo/ExecuTorchDemo"
10+
APP_PATH="executorch-examples/mv3/apple/ExecuTorchDemo/ExecuTorchDemo"
1111
MODEL_NAME="mv3"
1212
SIMULATOR_NAME="executorch"
1313

@@ -34,6 +34,10 @@ say() {
3434
echo -e "\033[1m\n\t** $1 **\n\033[0m"
3535
}
3636

37+
say "Cloning the Demo App"
38+
39+
git clone --depth 1 https://github.com/pytorch-labs/executorch-examples.git
40+
3741
say "Installing CoreML Backend Requirements"
3842

3943
./backends/apple/coreml/scripts/install_requirements.sh

‎.ci/scripts/test_llava.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ run_and_verify() {
154154
EXPECTED_PREFIX="ASSISTANT: image captures a basketball game in progress, with several players on the court. One of the players is dribbling the ball, while the others are in various"
155155
else
156156
# set the expected prefix to be the same as prompt because there's a bug in sdpa_with_kv_cache that causes <unk> tokens.
157-
EXPECTED_PREFIX="ASSISTANT:"
157+
EXPECTED_PREFIX="ASSISTANT: image"
158158
fi
159159
if [[ "${RESULT}" == *"${EXPECTED_PREFIX}"* ]]; then
160160
echo "Expected result prefix: ${EXPECTED_PREFIX}"

‎.github/release.yml

+96
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
# .github/release.yml
2+
3+
changelog:
4+
exclude:
5+
labels:
6+
- ignore-for-release
7+
categories:
8+
- title: Breaking Changes 🛠
9+
labels:
10+
- Semver-Major
11+
- breaking-change
12+
- title: API
13+
labels:
14+
- "release notes: api"
15+
- title: ARM
16+
labels:
17+
- "release notes: arm"
18+
- "module: arm"
19+
- "partner: arm"
20+
- title: NXP
21+
labels:
22+
- "release notes: nxp"
23+
- "module: nxp"
24+
- title: Exir
25+
labels:
26+
- "release notes: exir"
27+
- "module: exir"
28+
- title: Misc
29+
labels:
30+
- "release notes: misc"
31+
- title: Apple
32+
labels:
33+
- "release notes: apple"
34+
- "module: coreml"
35+
- "module: mps"
36+
- title: Android
37+
labels:
38+
- "module: android"
39+
- title: IOS
40+
labels:
41+
- "module: ios"
42+
- title: Build
43+
labels:
44+
- "release notes: build"
45+
- title: Vulkan
46+
labels:
47+
- "release notes: vulkan"
48+
- "module: vulkan"
49+
- title: Cadence
50+
labels:
51+
- "release notes: cadence"
52+
- "module: cadence"
53+
- title: Runtime
54+
labels:
55+
- "release notes: runtime"
56+
- "module: runtime"
57+
- title: XNNPACK
58+
labels:
59+
- "release notes: xnnpack"
60+
- "module: xnnpack"
61+
- title: Devtools
62+
labels:
63+
- "release notes: devtools"
64+
- "module: devtools"
65+
- title: Examples
66+
labels:
67+
- "release notes: examples"
68+
- title: LLM
69+
labels:
70+
- "module: llm"
71+
- title: Mediatek
72+
labels:
73+
- "release notes: mediatek"
74+
- "partner: mediatek"
75+
- title: Openvino
76+
labels:
77+
- "release notes: openvino"
78+
- title: Qualcomm
79+
labels:
80+
- "release notes: qualcomm"
81+
- "partner: qualcomm"
82+
- "module: qnn"
83+
- title: Training
84+
labels:
85+
- "release notes: training"
86+
- "module: training"
87+
- title: Quantization
88+
labels:
89+
- "release notes: quantization"
90+
- title: Ops & kernels
91+
labels:
92+
- "release notes: ops & kernels"
93+
- "module: kernels"
94+
- title: Other Changes
95+
labels:
96+
- "*"

‎.github/scripts/extract_benchmark_results.py

+13-3
Original file line numberDiff line numberDiff line change
@@ -341,14 +341,18 @@ def transform(
341341
benchmark_results: List,
342342
benchmark_config: Dict[str, str],
343343
job_name: str,
344+
job_report: Any = {},
344345
) -> List:
345346
"""
346347
Transform the benchmark results into the format writable into the benchmark database
347348
"""
348349
# Overwrite the device name here with the job name as it has more information about
349350
# the device, i.e. Samsung Galaxy S22 5G instead of just Samsung
350351
for r in benchmark_results:
351-
r["deviceInfo"]["device"] = job_name
352+
is_private_device = job_report.get("is_private_instance", False)
353+
r["deviceInfo"]["device"] = (
354+
f"{job_name} (private)" if is_private_device else job_name
355+
)
352356

353357
# From https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
354358
return [
@@ -361,6 +365,8 @@ def transform(
361365
# Just keep a copy of the benchmark config here
362366
"benchmark_config": json.dumps(benchmark_config),
363367
"job_conclusion": "SUCCESS",
368+
"job_arn": job_report.get("arn", ""),
369+
"instance_arn": job_report.get("instance_arn", ""),
364370
},
365371
},
366372
"model": {
@@ -446,6 +452,7 @@ def transform_failure_record(
446452
"app_type": app_type,
447453
"job_conclusion": result,
448454
"failure_type": level,
455+
"job_arn": report.get("arn", ""),
449456
"job_report": json.dumps(report),
450457
},
451458
},
@@ -512,6 +519,7 @@ def get_benchmark_config(
512519
def extract_benchmark_result_from_artifact(
513520
artifact: Dict[str, Any],
514521
benchmark_config: Dict[str, str],
522+
job_report: Any,
515523
) -> List[Any]:
516524
job_name = artifact.get("job_name", "")
517525
artifact_type = artifact.get("type", "")
@@ -532,7 +540,9 @@ def extract_benchmark_result_from_artifact(
532540
)
533541
if not benchmark_results:
534542
return []
535-
return transform(app_type, benchmark_results, benchmark_config, job_name)
543+
return transform(
544+
app_type, benchmark_results, benchmark_config, job_name, job_report
545+
)
536546

537547

538548
def get_app_type(type: str):
@@ -674,7 +684,7 @@ def process_benchmark_results(content: Any, app: str, benchmark_configs: str):
674684
for job_artifact in job_artifacts:
675685
# generate result for each schema
676686
results = extract_benchmark_result_from_artifact(
677-
job_artifact, benchmark_config
687+
job_artifact, benchmark_config, job_report
678688
)
679689
all_benchmark_results.extend(results)
680690
return all_benchmark_results

‎.github/workflows/_android.yml

+10-4
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,18 @@ jobs:
1414
with:
1515
runner: linux.2xlarge
1616
docker-image: executorch-ubuntu-22.04-clang12-android
17-
submodules: 'true'
17+
submodules: 'recursive'
1818
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
1919
timeout: 90
2020
upload-artifact: android-apps
2121
upload-artifact-to-s3: true
2222
script: |
2323
set -eux
2424
25+
# Use sccache for NDK compiler as well
26+
export CMAKE_CXX_COMPILER_LAUNCHER=sccache
27+
export CMAKE_C_COMPILER_LAUNCHER=sccache
28+
2529
# The generic Linux job chooses to use base env, not the one setup by the image
2630
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
2731
conda activate "${CONDA_ENV}"
@@ -36,8 +40,9 @@ jobs:
3640
cp ${BUILD_AAR_DIR}/executorch.aar $ARTIFACTS_DIR_NAME
3741
3842
mkdir -p ${ARTIFACTS_DIR_NAME}/library_test_dir
39-
bash .ci/scripts/build_android_instrumentation.sh
40-
cp ${BUILD_AAR_DIR}/executorch_android/build/outputs/apk/androidTest/debug/executorch_android-debug-androidTest.apk "${ARTIFACTS_DIR_NAME}/library_test_dir"
43+
bash extension/android/executorch_android/android_test_setup.sh
44+
(cd extension/android; ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew :executorch_android:assembleAndroidTest)
45+
cp extension/android/executorch_android/build/outputs/apk/androidTest/debug/executorch_android-debug-androidTest.apk "${ARTIFACTS_DIR_NAME}/library_test_dir"
4146
4247
mkdir -p ${ARTIFACTS_DIR_NAME}/fp32-xnnpack-custom
4348
bash examples/models/llama/install_requirements.sh
@@ -130,7 +135,8 @@ jobs:
130135
# https://github.com/ReactiveCircus/android-emulator-runner. The max number
131136
# of cores we can set is 6, any higher number will be reduced to 6.
132137
cores: 6
133-
ram-size: 12288M
138+
ram-size: 16384M
139+
heap-size: 12288M
134140
force-avd-creation: false
135141
disable-animations: true
136142
emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none

‎.github/workflows/_unittest.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ jobs:
3333
with:
3434
runner: linux.2xlarge
3535
docker-image: ${{ inputs.docker-image }}
36-
submodules: 'true'
36+
submodules: 'recursive'
3737
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
3838
timeout: 90
3939
script: |
@@ -45,7 +45,7 @@ jobs:
4545
with:
4646
runner: macos-m1-stable
4747
python-version: '3.11'
48-
submodules: 'true'
48+
submodules: 'recursive'
4949
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
5050
script: |
5151
set -eux

0 commit comments

Comments
 (0)