diff --git a/lazy_player/__init__.py b/lazy_player/__init__.py
index 3518372..609685f 100644
--- a/lazy_player/__init__.py
+++ b/lazy_player/__init__.py
@@ -30,12 +30,16 @@ class MainWindow(Gtk.ApplicationWindow):
     overlay_hide_time: float
     last_position_save: float
 
+    directory_history: list[Path]
+    selection_history: dict[str, int]
+
     def __init__(self, *args: Any, thumbnailer: Thumbnailer, **kwargs: Any):
         super().__init__(*args, **kwargs)
         self.thumbnailer = thumbnailer
 
         # Directory history stack
-        self.directory_history: list[Path] = []
+        self.directory_history = []
+        self.selection_history = {}
 
         # For overlay text timeout
         self.overlay_hide_time = 0.0
@@ -199,10 +203,9 @@ class MainWindow(Gtk.ApplicationWindow):
         return frame_clock.get_frame_time() / 1_000_000
 
     @property
-    def selection(self) -> FileItem:
+    def selection(self) -> FileItem | None:
         selected_item = self.selection_model.get_selected_item()
-        assert selected_item
-        return cast(FileItem, selected_item)
+        return cast(FileItem, selected_item) if selected_item else None
 
     def _setup_list_item(self, factory: Gtk.SignalListItemFactory, list_item: Gtk.ListItem):
         # Create horizontal box to hold icon and label
@@ -247,55 +250,36 @@ class MainWindow(Gtk.ApplicationWindow):
 
         label.set_text(item.name)
         item.connect("notify::saved-position", update_icon)
+        item.connect("notify::saved-duration", update_icon)
         update_icon()
 
     def _refresh(self):
-        selected = self.selection.full_path
         self._populate_file_list()
 
-        for i in range(self.list_model.get_n_items()):
-            item = self.list_model.get_item(i)
-            if not item:
-                continue
-
-            file_item = cast(FileItem, item)
-
-            if file_item.full_path == selected:
-                self.list_view.scroll_to(
-                    i, Gtk.ListScrollFlags.SELECT | Gtk.ListScrollFlags.FOCUS, None
-                )
-                break
+        pos = self.selection_history.get(str(os.getcwd()), 0)
+        self.selection_model.set_selected(pos)
+        self.list_view.scroll_to(pos, Gtk.ListScrollFlags.SELECT | Gtk.ListScrollFlags.FOCUS, None)
 
     def _navigate_to(self, path: Path):
         self.directory_history.append(Path(os.getcwd()))
         os.chdir(path)
         self._populate_file_list()
 
+        pos = self.selection_history.get(str(path), 0)
+        self.selection_model.set_selected(pos)
+        self.list_view.scroll_to(pos, Gtk.ListScrollFlags.SELECT | Gtk.ListScrollFlags.FOCUS, None)
+
     def _navigate_back(self):
         if not self.directory_history:
             return
 
         prev_dir = self.directory_history.pop()
-        current_dir = Path(os.getcwd())
         os.chdir(prev_dir)
         self._populate_file_list()
 
-        # Find and select the directory we came from
-        for i in range(self.list_model.get_n_items()):
-            item = self.list_model.get_item(i)
-            if not item:
-                continue
-
-            file_item = cast(FileItem, item)
-
-            if file_item.file_type != FileType.DIRECTORY:
-                continue
-
-            if current_dir == file_item.full_path:
-                self.list_view.scroll_to(
-                    i, Gtk.ListScrollFlags.SELECT | Gtk.ListScrollFlags.FOCUS, None
-                )
-                break
+        pos = self.selection_history.get(str(prev_dir), 0)
+        self.selection_model.set_selected(pos)
+        self.list_view.scroll_to(pos, Gtk.ListScrollFlags.SELECT | Gtk.ListScrollFlags.FOCUS, None)
 
     def _on_selection_changed(
         self,
@@ -303,9 +287,15 @@ class MainWindow(Gtk.ApplicationWindow):
         position: int,
         n_items: int,
     ):
-        if selection_model.get_selected() != Gtk.INVALID_LIST_POSITION:
-            file_item = self.selection
+        position = selection_model.get_selected()
 
+        if position == Gtk.INVALID_LIST_POSITION:
+            return
+
+        self.selection_history[os.getcwd()] = position
+
+        file_item = self.selection
+        if file_item is not None:
             # Update thumbnail if available
             if file_item.thumbnail:
                 gbytes = GLib.Bytes.new(cast(Any, file_item.thumbnail))
@@ -369,7 +359,9 @@ class MainWindow(Gtk.ApplicationWindow):
                 else:
                     self.show_overlay_text("Subtitles turned off")
 
-                self.selection.saved_subtitle_track = index - 1
+                file_item = self.selection
+                if file_item is not None:
+                    file_item.saved_subtitle_track = index - 1
             else:
                 self.show_overlay_text("No subtitles available")
 
@@ -380,7 +372,9 @@ class MainWindow(Gtk.ApplicationWindow):
 
             if has_audio:
                 self.show_overlay_text(f"Audio #{index} ({lang})")
-                self.selection.saved_audio_track = index - 1
+                file_item = self.selection
+                if file_item is not None:
+                    file_item.saved_audio_track = index - 1
             else:
                 self.show_overlay_text("No audio tracks available")
 
@@ -393,7 +387,7 @@ class MainWindow(Gtk.ApplicationWindow):
 
         file_item = self.selection
 
-        if file_item.file_type == FileType.DIRECTORY:
+        if file_item is None or file_item.file_type == FileType.DIRECTORY:
             return
 
         position = file_item.saved_position
@@ -454,7 +448,7 @@ class MainWindow(Gtk.ApplicationWindow):
 
         file_item = self.selection
 
-        if file_item.file_type == FileType.DIRECTORY:
+        if file_item is None or file_item.file_type == FileType.DIRECTORY:
             return
 
         position = self.video_player.get_position()
@@ -489,10 +483,12 @@ class MainWindow(Gtk.ApplicationWindow):
 
         # Update thumbnail if available
         file_item = self.selection
-        if file_item.thumbnail and not self.thumbnail_image.get_paintable():
-            gbytes = GLib.Bytes.new(cast(Any, file_item.thumbnail))
-            texture = Gdk.Texture.new_from_bytes(gbytes)
-            self.thumbnail_image.set_paintable(texture)
+
+        if file_item is not None:
+            if file_item.thumbnail and not self.thumbnail_image.get_paintable():
+                gbytes = GLib.Bytes.new(cast(Any, file_item.thumbnail))
+                texture = Gdk.Texture.new_from_bytes(gbytes)
+                self.thumbnail_image.set_paintable(texture)
 
         return True
 
diff --git a/lazy_player/file_model.py b/lazy_player/file_model.py
index ae49f2c..546f4da 100644
--- a/lazy_player/file_model.py
+++ b/lazy_player/file_model.py
@@ -44,7 +44,7 @@ class FileItem(GObject.Object):
 
     @GObject.Property(type=GObject.TYPE_UINT64)
     def saved_duration(self):
-        return self._load_attribute("duration", 1) or 1
+        return self._load_attribute("duration", 60) or 60
 
     @saved_duration.setter
     def set_saved_duration(self, value: int):