diff --git a/lazy_player/__init__.py b/lazy_player/__init__.py
index e198c0a..b467ddf 100644
--- a/lazy_player/__init__.py
+++ b/lazy_player/__init__.py
@@ -377,8 +377,18 @@ class MainWindow(Gtk.ApplicationWindow):
         elif keyval == Gdk.keyval_from_name("BackSpace"):
             if self.directory_history:
                 prev_dir = self.directory_history.pop()
+                current_dir_name = Path(os.getcwd()).name
                 os.chdir(prev_dir)
                 self._populate_file_list()
+
+                # Find and select the directory we came from
+                for i in range(self.list_store.get_n_items()):
+                    item = self.list_store.get_item(i)
+                    if item and cast(FileItem, item).name == current_dir_name:
+                        self.list_view.scroll_to(
+                            i, Gtk.ListScrollFlags.SELECT | Gtk.ListScrollFlags.FOCUS, None
+                        )
+                        break
             return True
 
         return False