Move MLIR python sources to mlir/python.

* NFC but has some fixes for CMake glitches discovered along the way (things not cleaning properly, co-mingled depends).
* Includes previously unsubmitted fix in D98681 and a TODO to fix it more appropriately in a smaller followup.

Differential Revision: https://reviews.llvm.org/D101493

Author: stellaraccident

mlir/CMakeLists.txt

@@ -146,6 +146,11 @@ endif()
 # Generally things after this point may depend on MLIR_ALL_LIBS or libMLIR.so.
 add_subdirectory(tools)
 
+if(MLIR_BINDINGS_PYTHON_ENABLED)
+  # Python sources: built extensions come in via lib/Bindings/Python
+  add_subdirectory(python)
+endif()
+
 if( LLVM_INCLUDE_EXAMPLES )
   add_subdirectory(examples)
 endif()

mlir/cmake/modules/AddMLIRPythonExtension.cmake renamed to mlir/cmake/modules/AddMLIRPython.cmake

@@ -156,6 +156,7 @@ function(add_mlir_dialect_python_bindings tblgen_target)
   add_custom_command(
     TARGET ${tblgen_target} POST_BUILD
     COMMENT "Copying generated python source \"dialects/${dialect_filename}\""
+    BYPRODUCTS "${PROJECT_BINARY_DIR}/python/mlir/dialects/${dialect_filename}"
     COMMAND "${CMAKE_COMMAND}" -E copy_if_different
       "${CMAKE_CURRENT_BINARY_DIR}/${dialect_filename}"
       "${PROJECT_BINARY_DIR}/python/mlir/dialects/${dialect_filename}")

mlir/cmake/modules/CMakeLists.txt

@@ -81,7 +81,7 @@ if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
   install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/MLIRConfig.cmake
     ${CMAKE_CURRENT_SOURCE_DIR}/AddMLIR.cmake
-    ${CMAKE_CURRENT_SOURCE_DIR}/AddMLIRPythonExtension.cmake
+    ${CMAKE_CURRENT_SOURCE_DIR}/AddMLIRPython.cmake
     ${CMAKE_CURRENT_SOURCE_DIR}/MLIRDetectPythonEnv.cmake
     DESTINATION ${MLIR_INSTALL_PACKAGE_DIR}
     COMPONENT mlir-cmake-exports)

mlir/docs/Bindings/Python.md

@@ -1,77 +1,6 @@
-include(AddMLIRPythonExtension)
+include(AddMLIRPython)
 add_custom_target(MLIRBindingsPythonExtension)
 
-################################################################################
-# Copy python source tree.
-################################################################################
-
-file(GLOB_RECURSE PY_SRC_FILES
-  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
-  "${CMAKE_CURRENT_SOURCE_DIR}/mlir/*.py")
-
-add_custom_target(MLIRBindingsPythonSources ALL
-  DEPENDS ${PY_SRC_FILES}
-)
-add_dependencies(MLIRBindingsPythonExtension MLIRBindingsPythonSources)
-
-foreach(PY_SRC_FILE ${PY_SRC_FILES})
-  set(PY_DEST_FILE "${PROJECT_BINARY_DIR}/python/${PY_SRC_FILE}")
-  get_filename_component(PY_DEST_DIR "${PY_DEST_FILE}" DIRECTORY)
-  file(MAKE_DIRECTORY "${PY_DEST_DIR}")
-  add_custom_command(
-    TARGET MLIRBindingsPythonSources PRE_BUILD
-    COMMENT "Copying python source ${PY_SRC_FILE} -> ${PY_DEST_FILE}"
-    DEPENDS "${PY_SRC_FILE}"
-    COMMAND "${CMAKE_COMMAND}" -E create_symlink
-        "${CMAKE_CURRENT_SOURCE_DIR}/${PY_SRC_FILE}" "${PY_DEST_FILE}"
-  )
-endforeach()
-
-################################################################################
-# Generate dialect-specific bindings.
-################################################################################
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonAsyncOps
-  TD_FILE AsyncOps.td
-  DIALECT_NAME async_dialect)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonAsyncOps)
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonBuiltinOps
-  TD_FILE BuiltinOps.td
-  DIALECT_NAME builtin)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonBuiltinOps)
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonGPUOps
-  TD_FILE GPUOps.td
-  DIALECT_NAME gpu)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonGPUOps)
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonLinalgOps
-  TD_FILE LinalgOps.td
-  DIALECT_NAME linalg
-  DEPENDS LinalgOdsGen)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonLinalgOps)
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonMemRefOps
-  TD_FILE MemRefOps.td
-  DIALECT_NAME memref)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonMemRefOps)
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonShapeOps
-  TD_FILE ShapeOps.td
-  DIALECT_NAME shape)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonShapeOps)
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonStandardOps
-  TD_FILE StandardOps.td
-  DIALECT_NAME std)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonStandardOps)
-
-add_mlir_dialect_python_bindings(MLIRBindingsPythonTensorOps
-  TD_FILE TensorOps.td
-  DIALECT_NAME tensor)
-add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonTensorOps)
-
 ################################################################################
 # Build core python extension
 ################################################################################
@@ -92,42 +21,6 @@ add_mlir_python_extension(MLIRCoreBindingsPythonExtension _mlir
 )
 add_dependencies(MLIRBindingsPythonExtension MLIRCoreBindingsPythonExtension)
 
-# Note that we copy from the source tree just like for headers because
-# it will not be polluted with py_cache runtime artifacts (from testing and
-# such).
-install(
-  DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/mlir
-  DESTINATION python
-  COMPONENT MLIRBindingsPythonSources
-  FILES_MATCHING PATTERN "*.py"
-)
-
-if (NOT LLVM_ENABLE_IDE)
-  add_llvm_install_targets(
-    install-MLIRBindingsPythonSources
-    DEPENDS MLIRBindingsPythonSources
-    COMPONENT MLIRBindingsPythonSources)
-endif()
-
-# Dialect sources are generated. Install separately.
-# Note that __pycache__ directories may have been left by tests and other
-# executions. And __init__.py is handled as a regular source file.
-install(
-  DIRECTORY ${PROJECT_BINARY_DIR}/python/mlir/dialects
-  DESTINATION python/mlir
-  COMPONENT MLIRBindingsPythonDialects
-  FILES_MATCHING PATTERN "*.py"
-  PATTERN "__pycache__" EXCLUDE
-  PATTERN "__init__.py" EXCLUDE
-)
-
-if (NOT LLVM_ENABLE_IDE)
-  add_llvm_install_targets(
-    install-MLIRBindingsPythonDialects
-    DEPENDS MLIRBindingsPythonSources
-    COMPONENT MLIRBindingsPythonDialects)
-endif()
-
 add_subdirectory(Transforms)
 add_subdirectory(Conversions)
 

mlir/lib/Bindings/Python/CMakeLists.txt

@@ -0,0 +1,49 @@
+################################################################################
+# Copy python source tree.
+################################################################################
+
+file(GLOB_RECURSE PY_SRC_FILES
+  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
+  "${CMAKE_CURRENT_SOURCE_DIR}/mlir/*.py")
+
+add_custom_target(MLIRBindingsPythonSources ALL
+  DEPENDS
+    ${PY_SRC_FILES}
+)
+
+foreach(PY_SRC_FILE ${PY_SRC_FILES})
+  set(PY_DEST_FILE "${PROJECT_BINARY_DIR}/python/${PY_SRC_FILE}")
+  get_filename_component(PY_DEST_DIR "${PY_DEST_FILE}" DIRECTORY)
+  file(MAKE_DIRECTORY "${PY_DEST_DIR}")
+  add_custom_command(
+    TARGET MLIRBindingsPythonSources PRE_BUILD
+    COMMENT "Copying python source ${PY_SRC_FILE} -> ${PY_DEST_FILE}"
+    DEPENDS "${PY_SRC_FILE}"
+    BYPRODUCTS "${PY_DEST_FILE}"
+    COMMAND "${CMAKE_COMMAND}" -E create_symlink
+        "${CMAKE_CURRENT_SOURCE_DIR}/${PY_SRC_FILE}" "${PY_DEST_FILE}"
+  )
+endforeach()
+
+# Note that we copy from the source tree just like for headers because
+# it will not be polluted with py_cache runtime artifacts (from testing and
+# such).
+install(
+  DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/mlir
+  DESTINATION python
+  COMPONENT MLIRBindingsPythonSources
+  FILES_MATCHING PATTERN "*.py"
+)
+
+if (NOT LLVM_ENABLE_IDE)
+  add_llvm_install_targets(
+    install-MLIRBindingsPythonSources
+    DEPENDS MLIRBindingsPythonSources
+    COMPONENT MLIRBindingsPythonSources)
+endif()
+
+################################################################################
+# Generated sources.
+################################################################################
+
+add_subdirectory(mlir/dialects)

mlir/lib/Bindings/Python/.style.yapf renamed to mlir/python/.style.yapf

@@ -0,0 +1,71 @@
+include(AddMLIRPython)
+
+################################################################################
+# Generate dialect-specific bindings.
+################################################################################
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonAsyncOps
+  TD_FILE AsyncOps.td
+  DIALECT_NAME async_dialect)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonAsyncOps)
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonBuiltinOps
+  TD_FILE BuiltinOps.td
+  DIALECT_NAME builtin)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonBuiltinOps)
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonGPUOps
+  TD_FILE GPUOps.td
+  DIALECT_NAME gpu)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonGPUOps)
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonLinalgOps
+  TD_FILE LinalgOps.td
+  DIALECT_NAME linalg
+  DEPENDS LinalgOdsGen)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonLinalgOps)
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonMemRefOps
+  TD_FILE MemRefOps.td
+  DIALECT_NAME memref)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonMemRefOps)
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonShapeOps
+  TD_FILE ShapeOps.td
+  DIALECT_NAME shape)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonShapeOps)
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonStandardOps
+  TD_FILE StandardOps.td
+  DIALECT_NAME std)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonStandardOps)
+
+add_mlir_dialect_python_bindings(MLIRBindingsPythonTensorOps
+  TD_FILE TensorOps.td
+  DIALECT_NAME tensor)
+add_dependencies(MLIRBindingsPythonSources MLIRBindingsPythonTensorOps)
+
+################################################################################
+# Installation.
+################################################################################
+
+# Dialect sources are generated. Install separately.
+# Note that __pycache__ directories may have been left by tests and other
+# executions. And __init__.py is handled as a regular source file.
+# TODO: Eliminate this glob install, instead adding INSTALL_COMPONENT to
+# add_mlir_dialect_python_bindings and installing the precise file there.
+install(
+  DIRECTORY ${PROJECT_BINARY_DIR}/python/mlir/dialects
+  DESTINATION python/mlir
+  COMPONENT MLIRBindingsPythonDialects
+  FILES_MATCHING PATTERN "_*_gen.py"
+  PATTERN "__pycache__" EXCLUDE
+  PATTERN "__init__.py" EXCLUDE
+)
+
+if (NOT LLVM_ENABLE_IDE)
+  add_llvm_install_targets(
+    install-MLIRBindingsPythonDialects
+    DEPENDS MLIRBindingsPythonSources
+    COMPONENT MLIRBindingsPythonDialects)
+endif()

mlir/python/CMakeLists.txt

@@ -1,9 +1,12 @@
-add_subdirectory(Bindings)
 add_subdirectory(CAPI)
 add_subdirectory(EDSC)
 add_subdirectory(SDBM)
 add_subdirectory(lib)
 
+if(MLIR_BINDINGS_PYTHON_ENABLED)
+  add_subdirectory(python)
+endif()
+
 llvm_canonicalize_cmake_booleans(
   MLIR_BINDINGS_PYTHON_ENABLED
   LLVM_BUILD_EXAMPLES
@@ -111,6 +114,7 @@ endif()
 if(MLIR_BINDINGS_PYTHON_ENABLED)
   list(APPEND MLIR_TEST_DEPENDS
     MLIRBindingsPythonExtension
+    MLIRBindingsPythonSources
     MLIRBindingsPythonTestOps
     MLIRTransformsBindingsPythonExtension
     MLIRConversionsBindingsPythonExtension

mlir/lib/Bindings/Python/mlir/_cext_loader.py renamed to mlir/python/mlir/_cext_loader.py

@@ -1,4 +1,5 @@
-include(AddMLIRPythonExtension)
+include(AddMLIRPython)
+
 add_mlir_dialect_python_bindings(MLIRBindingsPythonTestOps
   TD_FILE python_test_ops.td
   DIALECT_NAME python_test)