Skip to content

Commit 85d85c7

Browse files
jackzhxngjackzhxng
committed
Merge branch 'main' into jz/hf-download
2 parents 4db8f07 + cb583f3 commit 85d85c7

File tree

13 files changed

+206
-58
lines changed

13 files changed

+206
-58
lines changed

‎.ci/scripts/test_llama_torchao_lowbit.sh

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,6 @@ ${PYTHON_EXECUTABLE} -m examples.models.llama.export_llama \
7878
-qmode "torchao:8da${QLINEAR_BITWIDTH}w" \
7979
--group_size ${QLINEAR_GROUP_SIZE} \
8080
-E "torchao:${QEMBEDDING_BITWIDTH},${QEMBEDDING_GROUP_SIZE}" \
81-
--disable_dynamic_shape \
8281
-d fp32
8382

8483
# Test run

‎backends/apple/coreml/CMakeLists.txt

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# Copyright © 2023 Apple Inc. All rights reserved.
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
27

38
cmake_minimum_required(VERSION 3.19)
49

@@ -111,32 +116,48 @@ set(PROTOBUF_SOURCES
111116
runtime/sdk/format/WordTagger.pb.cc
112117
)
113118

119+
find_library(FOUNDATION_FRAMEWORK Foundation)
120+
121+
# CoreML util
122+
add_library(coreml_util ${UTIL_SOURCES})
123+
target_include_directories(coreml_util PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/runtime/util)
124+
target_link_libraries(coreml_util PRIVATE ${FOUNDATION_FRAMEWORK})
125+
126+
install(
127+
TARGETS coreml_util
128+
DESTINATION lib
129+
INCLUDES
130+
DESTINATION ${_common_include_directories}
131+
)
132+
133+
# CoreML inmemoryfs
134+
add_library(coreml_inmemoryfs ${INMEMORYFS_SOURCES})
135+
target_include_directories(coreml_inmemoryfs PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/runtime/inmemoryfs)
136+
target_link_libraries(coreml_inmemoryfs PRIVATE coreml_util ${FOUNDATION_FRAMEWORK})
137+
138+
install(
139+
TARGETS coreml_inmemoryfs
140+
DESTINATION lib
141+
INCLUDES
142+
DESTINATION ${_common_include_directories}
143+
)
144+
114145
# Define the delegate library
115146
add_library(coremldelegate)
116-
target_sources(
117-
coremldelegate PRIVATE ${INMEMORYFS_SOURCES} ${KVSTORE_SOURCES}
118-
${DELEGATE_SOURCES} ${UTIL_SOURCES}
119-
)
147+
target_sources(coremldelegate PRIVATE ${KVSTORE_SOURCES} ${DELEGATE_SOURCES})
120148

121149
target_include_directories(
122150
coremldelegate PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/runtime/include
123151
)
124152
target_include_directories(
125153
coremldelegate PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/runtime/kvstore
126154
)
127-
target_include_directories(
128-
coremldelegate PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/runtime/inmemoryfs
129-
)
130155
target_include_directories(
131156
coremldelegate PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/runtime/delegate
132157
)
133-
target_include_directories(
134-
coremldelegate PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/runtime/util
135-
)
136158
target_include_directories(coremldelegate PRIVATE ${EXECUTORCH_ROOT}/..)
137159
target_include_directories(coremldelegate PRIVATE ${EXECUTORCH_ROOT}/runtime/core/portable_type/c10)
138160
target_compile_definitions(coremldelegate PRIVATE C10_USING_CUSTOM_GENERATED_MACROS)
139-
target_link_libraries(coremldelegate PRIVATE executorch_core)
140161

141162
if(EXECUTORCH_BUILD_DEVTOOLS)
142163
target_sources(coremldelegate PRIVATE ${SDK_SOURCES} ${PROTOBUF_SOURCES})
@@ -156,13 +177,17 @@ endif()
156177

157178
find_library(ACCELERATE_FRAMEWORK Accelerate)
158179
find_library(COREML_FRAMEWORK CoreML)
159-
find_library(FOUNDATION_FRAMEWORK Foundation)
160180
find_library(SQLITE_LIBRARY sqlite3)
161181

162182
target_link_libraries(
163183
coremldelegate
164-
PRIVATE executorch_core ${ACCELERATE_FRAMEWORK} ${COREML_FRAMEWORK}
165-
${FOUNDATION_FRAMEWORK} ${SQLITE_LIBRARY}
184+
PUBLIC coreml_util
185+
coreml_inmemoryfs
186+
PRIVATE executorch_core
187+
${ACCELERATE_FRAMEWORK}
188+
${COREML_FRAMEWORK}
189+
${FOUNDATION_FRAMEWORK}
190+
${SQLITE_LIBRARY}
166191
)
167192

168193
target_link_options_shared_lib(coremldelegate)

‎backends/vulkan/vulkan_preprocess.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
)
4848
from executorch.exir.backend.utils import DelegateMappingBuilder
4949

50-
from executorch.exir.memory_planning import greedy
50+
from executorch.exir.memory_planning import greedy, memory_planning_algorithm_suite
5151
from executorch.exir.pass_base import ExportPass, PassBase
5252

5353
from executorch.exir.passes import MemoryPlanningPass, SpecPropPass
@@ -199,11 +199,14 @@ def preprocess( # noqa: C901
199199
# Finally, apply dynamic shape passes and memory planning pass. These passes
200200
# must be applied only when the graph structure is finalized.
201201
greedy_memory_planning = partial(greedy, allow_overlapping_allocations=False)
202+
mem_planning_suite = partial(
203+
memory_planning_algorithm_suite, algo_list=[greedy_memory_planning]
204+
)
202205
program = apply_passes(
203206
program,
204207
[
205208
ConstraintBasedSymShapeEvalPass(),
206-
MemoryPlanningPass(memory_planning_algo=greedy_memory_planning),
209+
MemoryPlanningPass(memory_planning_algo=mem_planning_suite),
207210
],
208211
)
209212

‎examples/apple/coreml/executor_runner/coreml_executor_runner.xcodeproj/project.pbxproj

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
/* Begin PBXBuildFile section */
1010
38626BB42B225A560059413D /* libflatccrt.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 38626BB32B225A560059413D /* libflatccrt.a */; };
1111
38626BB52B225A890059413D /* libetdump.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 38626BAF2B21C98F0059413D /* libetdump.a */; };
12+
879121DA2D91DDBA001E6C66 /* libcoreml_inmemoryfs.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 879121D82D91DDBA001E6C66 /* libcoreml_inmemoryfs.a */; };
13+
879121DB2D91DDBA001E6C66 /* libcoreml_util.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 879121D92D91DDBA001E6C66 /* libcoreml_util.a */; };
1214
C94D51592ACF4BFC00AF47FD /* main.mm in Sources */ = {isa = PBXBuildFile; fileRef = C94D51582ACF4BFC00AF47FD /* main.mm */; };
1315
C94D515E2ACFCBA000AF47FD /* libexecutorch.a in Frameworks */ = {isa = PBXBuildFile; fileRef = C94D515C2ACFCBA000AF47FD /* libexecutorch.a */; };
1416
C94D51622ACFCBBA00AF47FD /* libsqlite3.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = C94D51612ACFCBBA00AF47FD /* libsqlite3.tbd */; };
@@ -36,6 +38,8 @@
3638
/* Begin PBXFileReference section */
3739
38626BAF2B21C98F0059413D /* libetdump.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libetdump.a; path = libraries/libetdump.a; sourceTree = "<group>"; };
3840
38626BB32B225A560059413D /* libflatccrt.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libflatccrt.a; path = "../../../../third-party/flatcc/lib/libflatccrt.a"; sourceTree = "<group>"; };
41+
879121D82D91DDBA001E6C66 /* libcoreml_inmemoryfs.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libcoreml_inmemoryfs.a; path = "libraries/libcoreml_inmemoryfs.a"; sourceTree = "<group>"; };
42+
879121D92D91DDBA001E6C66 /* libcoreml_util.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libcoreml_util.a; path = "libraries/libcoreml_util.a"; sourceTree = "<group>"; };
3943
C94D514E2ACF4B9300AF47FD /* coreml_executor_runner */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = coreml_executor_runner; sourceTree = BUILT_PRODUCTS_DIR; };
4044
C94D51582ACF4BFC00AF47FD /* main.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = main.mm; sourceTree = "<group>"; };
4145
C94D515C2ACFCBA000AF47FD /* libexecutorch.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libexecutorch.a; path = libraries/libexecutorch.a; sourceTree = "<group>"; };
@@ -54,6 +58,8 @@
5458
isa = PBXFrameworksBuildPhase;
5559
buildActionMask = 2147483647;
5660
files = (
61+
879121DA2D91DDBA001E6C66 /* libcoreml_inmemoryfs.a in Frameworks */,
62+
879121DB2D91DDBA001E6C66 /* libcoreml_util.a in Frameworks */,
5763
38626BB52B225A890059413D /* libetdump.a in Frameworks */,
5864
F24817E72BC65B2000E80D98 /* libexecutorch_core.a in Frameworks */,
5965
38626BB42B225A560059413D /* libflatccrt.a in Frameworks */,
@@ -91,6 +97,8 @@
9197
C94D51602ACFCBBA00AF47FD /* Frameworks */ = {
9298
isa = PBXGroup;
9399
children = (
100+
879121D82D91DDBA001E6C66 /* libcoreml_inmemoryfs.a */,
101+
879121D92D91DDBA001E6C66 /* libcoreml_util.a */,
94102
C988D69C2B998CD700979CF6 /* libprotobuf-lite.a */,
95103
38626BB32B225A560059413D /* libflatccrt.a */,
96104
38626BAF2B21C98F0059413D /* libetdump.a */,
@@ -214,7 +222,7 @@
214222
GCC_OPTIMIZATION_LEVEL = 0;
215223
GCC_PREPROCESSOR_DEFINITIONS = (
216224
"DEBUG=1",
217-
"C10_USING_CUSTOM_GENERATED_MACROS",
225+
C10_USING_CUSTOM_GENERATED_MACROS,
218226
"$(inherited)",
219227
);
220228
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
@@ -273,7 +281,7 @@
273281
GCC_C_LANGUAGE_STANDARD = gnu17;
274282
GCC_NO_COMMON_BLOCKS = YES;
275283
GCC_PREPROCESSOR_DEFINITIONS = (
276-
"C10_USING_CUSTOM_GENERATED_MACROS",
284+
C10_USING_CUSTOM_GENERATED_MACROS,
277285
"$(inherited)",
278286
);
279287
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
@@ -297,9 +305,9 @@
297305
DEVELOPMENT_TEAM = "";
298306
ENABLE_HARDENED_RUNTIME = YES;
299307
HEADER_SEARCH_PATHS = (
300-
"$(SRCROOT)/include",
301-
"$(SRCROOT)/include/executorch/runtime/core/portable_type/c10",
302-
);
308+
"$(SRCROOT)/include",
309+
"$(SRCROOT)/include/executorch/runtime/core/portable_type/c10",
310+
);
303311
IPHONEOS_DEPLOYMENT_TARGET = 16.0;
304312
LIBRARY_SEARCH_PATHS = (
305313
"$(SRCROOT)/libraries",
@@ -319,9 +327,9 @@
319327
DEVELOPMENT_TEAM = "";
320328
ENABLE_HARDENED_RUNTIME = YES;
321329
HEADER_SEARCH_PATHS = (
322-
"$(SRCROOT)/include",
323-
"$(SRCROOT)/include/executorch/runtime/core/portable_type/c10",
324-
);
330+
"$(SRCROOT)/include",
331+
"$(SRCROOT)/include/executorch/runtime/core/portable_type/c10",
332+
);
325333
IPHONEOS_DEPLOYMENT_TARGET = 16.0;
326334
LIBRARY_SEARCH_PATHS = (
327335
"$(SRCROOT)/libraries",

‎examples/apple/coreml/scripts/build_executor_runner.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ find "$CMAKE_BUILD_DIR_PATH/" -name 'libexecutorch_core.a' -exec cp -f "{}" "$LI
6666
find "$CMAKE_BUILD_DIR_PATH/" -name 'libprotobuf-lite.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH/libprotobuf-lite.a" \;
6767
find "$CMAKE_BUILD_DIR_PATH/" -name 'libprotobuf-lited.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH/libprotobuf-lite.a" \;
6868
find "$CMAKE_BUILD_DIR_PATH/" -name 'libetdump.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH/libetdump.a" \;
69+
find "$CMAKE_BUILD_DIR_PATH/" -name 'libcoreml_util.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH/libcoreml_util.a" \;
70+
find "$CMAKE_BUILD_DIR_PATH/" -name 'libcoreml_inmemoryfs.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH/libcoreml_inmemoryfs.a" \;
6971
find "$CMAKE_BUILD_DIR_PATH/" -name 'libcoremldelegate.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH/libcoremldelegate.a" \;
7072
find "$CMAKE_BUILD_DIR_PATH/" -name 'libportable_ops_lib.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH/libportable_ops_lib.a" \;
7173
find "$CMAKE_BUILD_DIR_PATH/" -name 'libportable_kernels.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH/libportable_kernels.a" \;

‎examples/models/llama/README.md

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -382,7 +382,7 @@ Please refer to [this tutorial](https://pytorch.org/executorch/main/llm/llama-de
382382
383383
## Running with low-bit kernels
384384
385-
We now give instructions for quantizating and running your model with low-bit kernels. These are still experimental, and require you do development on an Arm-based Mac. Also note that low-bit quantization often requires QAT (quantization-aware training) to give good quality results. Currently dynamic shapes must be disabled when exporting a model with these kernels.
385+
We now give instructions for quantizating and running your model with low-bit kernels. These are still experimental, and require you do development on an Arm-based Mac. Also note that low-bit quantization often requires QAT (quantization-aware training) to give good quality results.
386386
387387
First export your model for lowbit quantization (step 2 above):
388388
@@ -408,7 +408,6 @@ python -m examples.models.llama.export_llama \
408408
-qmode "torchao:8da${QLINEAR_BITWIDTH}w" \
409409
--group_size ${QLINEAR_GROUP_SIZE} \
410410
-E "torchao:${QEMBEDDING_BITWIDTH},${QEMBEDDING_GROUP_SIZE}" \
411-
--disable_dynamic_shape \
412411
-d fp32
413412
```
414413

‎examples/models/llama/export_llama_lib.py

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -723,19 +723,6 @@ def _validate_args(args):
723723
"Shared embedding is only supported with torchao quantization."
724724
)
725725

726-
if (
727-
args.quantization_mode is not None
728-
and args.quantization_mode.startswith("torchao:")
729-
) or (
730-
args.embedding_quantize is not None
731-
and args.embedding_quantize.startswith("torchao:")
732-
):
733-
if args.enable_dynamic_shape:
734-
raise ValueError(
735-
"Dynamic shape is not currently supported with torchao ops. Please use --disable_dynamic_shape."
736-
"If you need this feature, please file an issue."
737-
)
738-
739726

740727
def _to_edge_and_lower_llama_xnnpack(
741728
builder_exported,

0 commit comments

Comments
 (0)