feat: switch code formatting to ruff

-   Switch to `argparse` and clean up imports in `commands.py`.
-   Remove `requires-optional.txt` and `test_requirements/*`
    (put dependencies in `pyproject.toml` instead).
-   Remove `black` and use `ruff` instead.
-   Add `--noformat` option to `python commands.py codegen` for experimenting.
-   Pass output directory around to control what is formatted and linted.
-   Add `python commands.py format` to only do formatting.
-   Add `python commands.py lint` to check code.
-   Reformat comments in `codegen/*.py`.
-   Use double-quoted strings for (most) code generation.
-   Incorporate pandas version pinning from #5219.

Note:

1.  Only the generated code is formatted and linted:
    -   `plotly/validators/**/*.py`
    -   `plotly/graph_objs/**/*.py`
    -   `plotly/graph_objects/__init__.py`

2.  The strings in the data used by code generation are (for example)
    `"'some_name'"` (i.e., have embedded single quotes). This PR does
    not try to fix that: instead, we rely on reformatting to turn all
    the single-quoted strings into double-quoted strings.

Author: gvwilson

codegen/__init__.py

@@ -3,6 +3,7 @@
 import os.path as opath
 import shutil
 import subprocess
+import sys
 
 from codegen.datatypes import build_datatype_py, write_datatype_py
 from codegen.compatibility import (
@@ -26,10 +27,6 @@
     get_data_validator_instance,
 )
 
-# Target Python version for code formatting with Black.
-# Must be one of the values listed in pyproject.toml.
-BLACK_TARGET_VERSIONS = "py38 py39 py310 py311 py312"
-
 
 # Import notes
 # ------------
@@ -39,15 +36,14 @@
 # helpers that are only needed during code generation should reside in the
 # codegen/ package, and helpers used both during code generation and at
 # runtime should reside in the _plotly_utils/ package.
-# ----------------------------------------------------------------------------
+
 def preprocess_schema(plotly_schema):
     """
     Central location to make changes to schema before it's seen by the
     PlotlyNode classes
     """
 
     # Update template
-    # ---------------
     layout = plotly_schema["layout"]["layoutAttributes"]
 
     # Create codegen-friendly template scheme
@@ -89,33 +85,41 @@ def preprocess_schema(plotly_schema):
         items["colorscale"] = items.pop("concentrationscales")
 
 
-def perform_codegen(reformat=True):
-    # Set root codegen output directory
-    # ---------------------------------
-    # (relative to project root)
-    abs_file_path = opath.realpath(__file__)
-    project_root = opath.dirname(opath.dirname(abs_file_path))
-    outdir = opath.join(project_root, "plotly")
+def make_paths(outdir):
+    """Make various paths needed for formatting and linting."""
+
+    validators_dir = opath.join(outdir, "validators")
+    graph_objs_dir = opath.join(outdir, "graph_objs")
+    graph_objects_path = opath.join(outdir, "graph_objects", "__init__.py")
+    return validators_dir, graph_objs_dir, graph_objects_path
+
+
+def lint_code(outdir):
+    """Check Python code using settings in pyproject.toml."""
+
+    subprocess.call(["ruff", "check", *make_paths(outdir)])
+
+
+def reformat_code(outdir):
+    """Reformat Python code using settings in pyproject.toml."""
+
+    subprocess.call(["ruff", "format", *make_paths(outdir)])
+
+
+def perform_codegen(outdir, noformat=False):
+    """Generate code (and possibly reformat)."""
+
+    # Get paths
+    validators_dir, graph_objs_dir, graph_objects_path = make_paths(outdir)
 
     # Delete prior codegen output
-    # ---------------------------
-    validators_pkgdir = opath.join(outdir, "validators")
-    if opath.exists(validators_pkgdir):
-        shutil.rmtree(validators_pkgdir)
-
-    graph_objs_pkgdir = opath.join(outdir, "graph_objs")
-    if opath.exists(graph_objs_pkgdir):
-        shutil.rmtree(graph_objs_pkgdir)
-
-    # plotly/datatypes is not used anymore, but was at one point so we'll
-    # still delete it if we find it in case a developer is upgrading from an
-    # older version
-    datatypes_pkgdir = opath.join(outdir, "datatypes")
-    if opath.exists(datatypes_pkgdir):
-        shutil.rmtree(datatypes_pkgdir)
+    if opath.exists(validators_dir):
+        shutil.rmtree(validators_dir)
+    if opath.exists(graph_objs_dir):
+        shutil.rmtree(graph_objs_dir)
 
     # Load plotly schema
-    # ------------------
+    project_root = opath.dirname(outdir)
     plot_schema_path = opath.join(
         project_root, "codegen", "resources", "plot-schema.json"
     )
@@ -124,19 +128,17 @@ def perform_codegen(reformat=True):
         plotly_schema = json.load(f)
 
     # Preprocess Schema
-    # -----------------
     preprocess_schema(plotly_schema)
 
     # Build node lists
-    # ----------------
-    # ### TraceNode ###
+    # TraceNode
     base_traces_node = TraceNode(plotly_schema)
     compound_trace_nodes = PlotlyNode.get_all_compound_datatype_nodes(
         plotly_schema, TraceNode
     )
     all_trace_nodes = PlotlyNode.get_all_datatype_nodes(plotly_schema, TraceNode)
 
-    # ### LayoutNode ###
+    # LayoutNode
     compound_layout_nodes = PlotlyNode.get_all_compound_datatype_nodes(
         plotly_schema, LayoutNode
     )
@@ -155,14 +157,14 @@ def perform_codegen(reformat=True):
         if node.is_array_element and node.has_child("xref") and node.has_child("yref")
     ]
 
-    # ### FrameNode ###
+    # FrameNode
     compound_frame_nodes = PlotlyNode.get_all_compound_datatype_nodes(
         plotly_schema, FrameNode
     )
     frame_node = compound_frame_nodes[0]
     all_frame_nodes = PlotlyNode.get_all_datatype_nodes(plotly_schema, FrameNode)
 
-    # ### All nodes ###
+    # All nodes
     all_datatype_nodes = all_trace_nodes + all_layout_nodes + all_frame_nodes
 
     all_compound_nodes = [
@@ -172,37 +174,34 @@ def perform_codegen(reformat=True):
     ]
 
     # Write out validators
-    # --------------------
-    # # ### Layout ###
+
+    # # Layout
     for node in all_layout_nodes:
         write_validator_py(outdir, node)
 
-    # ### Trace ###
+    # Trace
     for node in all_trace_nodes:
         write_validator_py(outdir, node)
 
-    # ### Frames ###
+    # Frames
     for node in all_frame_nodes:
         write_validator_py(outdir, node)
 
-    # ### Data (traces) validator ###
+    # Data (traces) validator
     write_data_validator_py(outdir, base_traces_node)
 
     # Alls
-    # ----
     alls = {}
 
     # Write out datatypes
-    # -------------------
     for node in all_compound_nodes:
         write_datatype_py(outdir, node)
 
-    # ### Deprecated ###
+    # Deprecated
     # These are deprecated legacy datatypes like graph_objs.Marker
     write_deprecated_datatypes(outdir)
 
     # Write figure class to graph_objs
-    # --------------------------------
     data_validator = get_data_validator_instance(base_traces_node)
     layout_validator = layout_node.get_validator_instance()
     frame_validator = frame_node.get_validator_instance()
@@ -218,8 +217,7 @@ def perform_codegen(reformat=True):
     )
 
     # Write validator __init__.py files
-    # ---------------------------------
-    # ### Write __init__.py files for each validator package ###
+    # Write __init__.py files for each validator package
     validator_rel_class_imports = {}
     for node in all_datatype_nodes:
         if node.is_mapped:
@@ -239,7 +237,6 @@ def perform_codegen(reformat=True):
         write_init_py(validators_pkg, path_parts, [], rel_classes)
 
     # Write datatype __init__.py files
-    # --------------------------------
     datatype_rel_class_imports = {}
     datatype_rel_module_imports = {}
 
@@ -257,16 +254,16 @@ def perform_codegen(reformat=True):
                 f".{node.name_undercase}"
             )
 
-    # ### Write plotly/graph_objs/graph_objs.py ###
-    # This if for backward compatibility. It just imports everything from
+    # Write plotly/graph_objs/graph_objs.py
+    # This is for backward compatibility. It just imports everything from
     # graph_objs/__init__.py
     write_graph_objs_graph_objs(outdir)
 
-    # ### Add Figure and FigureWidget ###
+    # Add Figure and FigureWidget
     root_datatype_imports = datatype_rel_class_imports[()]
     root_datatype_imports.append("._figure.Figure")
 
-    # ### Add deprecations ###
+    # Add deprecations
     for dep_clas in DEPRECATED_DATATYPES:
         root_datatype_imports.append(f"._deprecations.{dep_clas}")
 
@@ -302,14 +299,14 @@ def __getattr__(import_name):
 
         return orig_getattr(import_name)
 """
-    # ### __all__ ###
+    # __all__
     for path_parts, class_names in alls.items():
         if path_parts and class_names:
             filepath = opath.join(outdir, "graph_objs", *path_parts, "__init__.py")
             with open(filepath, "at") as f:
                 f.write(f"\n__all__ = {class_names}")
 
-    # ### Output datatype __init__.py files ###
+    # Output datatype __init__.py files
     graph_objs_pkg = opath.join(outdir, "graph_objs")
     for path_parts in datatype_rel_class_imports:
         rel_classes = sorted(datatype_rel_class_imports[path_parts])
@@ -320,7 +317,7 @@ def __getattr__(import_name):
             init_extra = ""
         write_init_py(graph_objs_pkg, path_parts, rel_modules, rel_classes, init_extra)
 
-    # ### Output graph_objects.py alias
+    # Output graph_objects.py alias
     graph_objects_rel_classes = [
         "..graph_objs." + rel_path.split(".")[-1]
         for rel_path in datatype_rel_class_imports[()]
@@ -335,22 +332,19 @@ def __getattr__(import_name):
         graph_objects_rel_classes,
         init_extra=optional_figure_widget_import,
     )
-    graph_objects_path = opath.join(outdir, "graph_objects", "__init__.py")
     os.makedirs(opath.join(outdir, "graph_objects"), exist_ok=True)
     with open(graph_objects_path, "wt") as f:
         f.write(graph_objects_init_source)
 
-    # ### Run black code formatter on output directories ###
-    if reformat:
-        target_version = [
-            f"--target-version={v}" for v in BLACK_TARGET_VERSIONS.split()
-        ]
-        subprocess.call(["black", *target_version, validators_pkgdir])
-        subprocess.call(["black", *target_version, graph_objs_pkgdir])
-        subprocess.call(["black", *target_version, graph_objects_path])
-    else:
+    # Run black code formatter on output directories
+    if noformat:
         print("skipping reformatting")
+    else:
+        reformat_code(outdir)
 
 
 if __name__ == "__main__":
-    perform_codegen()
+    if len(sys.argv) != 2:
+        print("Usage: codegen [dirname]", file=sys.stderr)
+        sys.exit(1)
+    perform_codegen(sys.argv[1])

codegen/compatibility.py

@@ -52,28 +52,24 @@ def build_deprecated_datatypes_py():
     """
 
     # Initialize source code buffer
-    # -----------------------------
     buffer = StringIO()
 
     # Write warnings import
-    # ---------------------
     buffer.write("import warnings\n")
 
     # Write warnings filter
-    # ---------------------
     # Use filter to enable DeprecationWarnings on our deprecated classes
     buffer.write(
         r"""
-warnings.filterwarnings('default',
-                        r'plotly\.graph_objs\.\w+ is deprecated',
+warnings.filterwarnings("default",
+                        r"plotly\.graph_objs\.\w+ is deprecated",
                         DeprecationWarning)
 
 
 """
     )
 
     # Write deprecated class definitions
-    # ----------------------------------
     for class_name, opts in DEPRECATED_DATATYPES.items():
         base_class_name = opts["base_type"].__name__
         depr_msg = build_deprecation_message(class_name, **opts)
@@ -93,7 +89,6 @@ def __init__(self, *args, **kwargs):
         )
 
     # Return source string
-    # --------------------
     return buffer.getvalue()
 
 
@@ -175,12 +170,10 @@ def write_deprecated_datatypes(outdir):
     None
     """
     # Generate source code
-    # --------------------
     datatype_source = build_deprecated_datatypes_py()
     filepath = opath.join(outdir, "graph_objs", "_deprecations.py")
 
     # Write file
-    # ----------
     write_source_py(datatype_source, filepath)