Benchmark: Enable 9.6, 9.8 (#4118)

Fix bench for newer ghc versions
The following have been done:
1. No longer use the implicit-hie to generate the hie.yaml for the bench examples and in favor of using "cradle:\n cabal:\n", seems to be working with modern cabal.
2. upgrade benchmark to use 9.6, 9.8 (The latest two we support for now).
3. upgrade bench examples to `Cabal version: 3.10.2.1, lsp-types version: 2.1.1.0`
4. fix minor error that `*.hp` files duplicates its extension name

---------

Co-authored-by: fendor <[email protected]>

Author: soulomoon

.github/workflows/bench.yml

@@ -46,9 +46,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # benching the two latest GHCs we support now
+        # since benchmark are expansive.
+        # choosing the two latest are easier to maintain and more forward looking
+        # see discussion https://github.com/haskell/haskell-language-server/pull/4118
+        # also possible to add more GHCs if we performs better in the future.
         ghc:
-          - '9.2'
-          - '9.4'
+          - '9.6'
+          - '9.8'
         os:
           - ubuntu-latest
 
@@ -115,7 +120,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ghc: ['9.2', '9.4']
+        ghc: ['9.6', '9.8']
         os: [ubuntu-latest]
         cabal: ['3.10']
         example: ['cabal', 'lsp-types']

bench/config.yaml

@@ -21,18 +21,18 @@ examples:
   # Medium-sized project without TH
   - name: cabal
     package: Cabal
-    version: 3.6.3.0
+    version: 3.10.2.1
     modules:
         - src/Distribution/Simple.hs
-        - src/Distribution/Types/Module.hs
+        - src/Distribution/Types/ComponentLocalBuildInfo.hs
     extra-args: [] # extra HLS command line args
   # Small-sized project with TH
   - name: lsp-types
     package: lsp-types
-    version: 1.5.0.0
+    version: 2.1.1.0
     modules:
-        - src/Language/LSP/Types/WatchedFiles.hs
-        - src/Language/LSP/Types/CallHierarchy.hs
+        - src/Language/LSP/Protocol/Types/SemanticTokens.hs
+        - generated/Language/LSP/Protocol/Internal/Types/NotebookDocumentChangeEvent.hs
 
   - name: MultiLayerModules
     path: bench/MultiLayerModules.sh
@@ -129,7 +129,7 @@ versions:
 # WARNING: Currently bench versions later than e4234a3a5e347db249fccefb8e3fb36f89e8eafb
 # will be unable to send plugin configurations to earlier HLS versions. This causes
 # all plugins in those versions to always be enabled.
-# In addition bench proactively disables all plugins it knows about besides the 
+# In addition bench proactively disables all plugins it knows about besides the
 # ones in the following list. However because it can only disable plugins it
 # knows about, any plugins that are in old versions but were removed from HLS
 # before the current bench will not be disabled.

ghcide-bench/ghcide-bench.cabal

@@ -89,7 +89,6 @@ test-suite test
     default-language: GHC2021
     build-tool-depends:
         ghcide:ghcide,
-        implicit-hie:gen-hie
     main-is: Main.hs
     hs-source-dirs: test
     ghc-options: -Wunused-packages

ghcide-bench/src/Experiments.hs

@@ -594,15 +594,25 @@ callCommandLogging cmd = do
     output cmd
     callCommand cmd
 
+simpleCabalCradleContent :: String
+simpleCabalCradleContent = "cradle:\n  cabal:\n"
+
+simpleStackCradleContent :: String
+simpleStackCradleContent = "cradle:\n  stack:\n"
+
+-- | Setup the benchmark
+-- we need to create a hie.yaml file for the examples
+-- or the hie.yaml file would be searched in the parent directories recursively
+-- implicit-hie is error prone for the example test `lsp-types-2.1.1.0`
+-- we are using the simpleCabalCradleContent for the hie.yaml file instead.
+-- it works if we have cabal > 3.2.
 setup :: HasConfig => IO SetupResult
 setup = do
---   when alreadyExists $ removeDirectoryRecursive examplesPath
   benchDir <- case exampleDetails(example ?config) of
       ExamplePath examplePath -> do
           let hieYamlPath = examplePath </> "hie.yaml"
           alreadyExists <- doesFileExist hieYamlPath
-          unless alreadyExists $
-                cmd_ (Cwd examplePath) (FileStdout hieYamlPath) ("gen-hie"::String)
+          unless alreadyExists $ writeFile hieYamlPath simpleCabalCradleContent
           return examplePath
       ExampleScript examplePath' scriptArgs -> do
           let exampleDir = examplesPath </> exampleName (example ?config)
@@ -613,8 +623,8 @@ setup = do
             cmd_ (Cwd exampleDir) examplePath scriptArgs
             let hieYamlPath = exampleDir </> "hie.yaml"
             alreadyExists <- doesFileExist hieYamlPath
-            unless alreadyExists $
-                  cmd_ (Cwd exampleDir) (FileStdout hieYamlPath) ("gen-hie"::String)
+            unless alreadyExists $ writeFile hieYamlPath simpleCabalCradleContent
+
           return exampleDir
       ExampleHackage ExamplePackage{..} -> do
         let path = examplesPath </> package
@@ -627,7 +637,7 @@ setup = do
                 let cabalVerbosity = "-v" ++ show (fromEnum (verbose ?config))
                 callCommandLogging $ "cabal get " <> cabalVerbosity <> " " <> package <> " -d " <> examplesPath
                 let hieYamlPath = path </> "hie.yaml"
-                cmd_ (Cwd path) (FileStdout hieYamlPath) ("gen-hie"::String)
+                writeFile hieYamlPath simpleCabalCradleContent
                 -- Need this in case there is a parent cabal.project somewhere
                 writeFile
                     (path </> "cabal.project")
@@ -655,8 +665,7 @@ setup = do
                                 ,"compiler"]
                             ]
                         )
-
-                cmd_ (Cwd path) (FileStdout hieYamlPath) ("gen-hie"::String) ["--stack"::String]
+                writeFile hieYamlPath simpleStackCradleContent
         return path
 
   whenJust (shakeProfiling ?config) $ createDirectoryIfMissing True

haskell-language-server.cabal

@@ -1848,16 +1848,13 @@ test-suite wrapper-test
 benchmark benchmark
     import:           defaults, warnings
     -- Depends on shake-bench which is unbuildable after this point
-    if impl(ghc >= 9.5)
-      buildable: False
     type: exitcode-stdio-1.0
     ghc-options: -threaded
     main-is: Main.hs
     hs-source-dirs: bench
     build-tool-depends:
         ghcide-bench:ghcide-bench,
         hp2pretty:hp2pretty,
-        implicit-hie:gen-hie
     default-extensions:
         LambdaCase
         RecordWildCards

shake-bench/shake-bench.cabal

@@ -16,9 +16,6 @@ source-repository head
     location: https://github.com/haskell/haskell-language-server.git
 
 library
-  -- Depends on Chart which is unbuildable after this point
-  if impl(ghc >= 9.5)
-    buildable: False
   exposed-modules:  Development.Benchmark.Rules
   hs-source-dirs:   src
   build-depends:

shake-bench/src/Development/Benchmark/Rules.hs

@@ -333,7 +333,7 @@ benchRules build MkBenchRules{..} = do
              ++ concat
                 [[ "-h"
                   , "-i" <> show i
-                  , "-po" <> outHp
+                  , "-po" <> dropExtension outHp
                   , "-qg"]
                  | CheapHeapProfiling i <- [prof]]
              ++ ["-RTS"]