BUG: Index.difference not always returning a unique set of values (#55113)

* BUG: Index.difference not always returning a unique set of values

* whatsnew

* udpate test

* update test

Author: lukemanley

doc/source/whatsnew/v2.2.0.rst

@@ -262,7 +262,7 @@ Interval
 
 Indexing
 ^^^^^^^^
--
+- Bug in :meth:`Index.difference` not returning a unique set of values when ``other`` is empty or ``other`` is considered non-comparable (:issue:`55113`)
 -
 
 Missing

pandas/core/indexes/base.py

@@ -3598,14 +3598,14 @@ def difference(self, other, sort=None):
 
         if len(other) == 0:
             # Note: we do not (yet) sort even if sort=None GH#24959
-            result = self.rename(result_name)
+            result = self.unique().rename(result_name)
             if sort is True:
                 return result.sort_values()
             return result
 
         if not self._should_compare(other):
             # Nothing matches -> difference is everything
-            result = self.rename(result_name)
+            result = self.unique().rename(result_name)
             if sort is True:
                 return result.sort_values()
             return result

pandas/tests/indexes/test_setops.py

@@ -796,11 +796,21 @@ def test_difference_name_preservation(self, index, second_name, expected, sort):
             assert result.name == expected
 
     def test_difference_empty_arg(self, index, sort):
-        first = index[5:20]
+        first = index.copy()
+        first = first[5:20]
         first.name = "name"
         result = first.difference([], sort)
+        expected = index[5:20].unique()
+        expected.name = "name"
+        tm.assert_index_equal(result, expected)
 
-        tm.assert_index_equal(result, first)
+    def test_difference_should_not_compare(self):
+        # GH 55113
+        left = Index([1, 1])
+        right = Index([True])
+        result = left.difference(right)
+        expected = Index([1])
+        tm.assert_index_equal(result, expected)
 
     @pytest.mark.parametrize("index", ["string"], indirect=True)
     def test_difference_identity(self, index, sort):