diff --git a/lazy_player/__init__.py b/lazy_player/__init__.py
index b467ddf..dab78d0 100644
--- a/lazy_player/__init__.py
+++ b/lazy_player/__init__.py
@@ -151,10 +151,31 @@ class MainWindow(Gtk.ApplicationWindow):
                 file_item = cast(FileItem, selected_item)
 
                 if file_item.file_type == FileType.DIRECTORY:
-                    # Save current directory before changing
-                    self.directory_history.append(os.getcwd())
-                    os.chdir(file_item.full_path)
-                    self._populate_file_list()
+                    target_path = os.path.abspath(file_item.full_path)
+
+                    # Check if we have history and if target is where we came from
+                    if self.directory_history and os.path.samefile(
+                        os.path.dirname(target_path), self.directory_history[-1]
+                    ):
+                        # Use history instead
+                        prev_dir = self.directory_history.pop()
+                        current_dir_name = Path(os.getcwd()).name
+                        os.chdir(prev_dir)
+                        self._populate_file_list()
+
+                        # Find and select the directory we came from
+                        for i in range(self.list_store.get_n_items()):
+                            item = self.list_store.get_item(i)
+                            if item and cast(FileItem, item).name == current_dir_name:
+                                self.list_view.scroll_to(
+                                    i, Gtk.ListScrollFlags.SELECT | Gtk.ListScrollFlags.FOCUS, None
+                                )
+                                break
+                    else:
+                        # Regular directory navigation
+                        self.directory_history.append(os.getcwd())
+                        os.chdir(file_item.full_path)
+                        self._populate_file_list()
                     return
 
                 position = file_item.load_attribute("position", 0)