diff --git a/deluge/core/torrentmanager.py b/deluge/core/torrentmanager.py
index 26717a7f4..a88f97015 100644
--- a/deluge/core/torrentmanager.py
+++ b/deluge/core/torrentmanager.py
@@ -138,6 +138,7 @@ class TorrentManager(component.Component):
 
         # Create the torrents dict { torrent_id: Torrent }
         self.torrents = {}
+        self.queued_torrents = set()
 
         # This is a list of torrent_id when we shutdown the torrentmanager.
         # We use this list to determine if all active torrents have been paused
@@ -476,6 +477,9 @@ class TorrentManager(component.Component):
         if not options["add_paused"]:
             torrent.resume()
 
+        # Add to queued torrents set
+        self.queued_torrents.add(torrent.torrent_id)
+
         # Write the .torrent file to the state directory
         if filedump:
             try:
@@ -584,6 +588,10 @@ class TorrentManager(component.Component):
         # Stop the looping call
         self.torrents[torrent_id].prev_status_cleanup_loop.stop()
 
+        # Remove from set if it wasn't finished
+        if not self.torrents[torrent_id].is_finished:
+            self.queued_torrents.remove(torrent_id)
+
         # Remove the torrent from deluge's session
         try:
             del self.torrents[torrent_id]
@@ -816,7 +824,7 @@ class TorrentManager(component.Component):
 
     def queue_down(self, torrent_id):
         """Queue torrent down one position"""
-        if self.torrents[torrent_id].get_queue_position() == (len(self.torrents) - 1):
+        if self.torrents[torrent_id].get_queue_position() == (len(self.queued_torrents) - 1):
             return False
 
         self.torrents[torrent_id].handle.queue_position_down()
@@ -824,7 +832,7 @@ class TorrentManager(component.Component):
 
     def queue_bottom(self, torrent_id):
         """Queue torrent to bottom"""
-        if self.torrents[torrent_id].get_queue_position() == (len(self.torrents) - 1):
+        if self.torrents[torrent_id].get_queue_position() == (len(self.queued_torrents) - 1):
             return False
 
         self.torrents[torrent_id].handle.queue_position_bottom()
@@ -880,6 +888,9 @@ class TorrentManager(component.Component):
         torrent.is_finished = True
         torrent.update_state()
 
+        # Torrent is no longer part of the queue
+        self.queued_torrents.remove(torrent_id)
+
         # Only save resume data if it was actually downloaded something. Helps
         # on startup with big queues with lots of seeding torrents. Libtorrent
         # emits alert_torrent_finished for them, but there seems like nothing
@@ -1010,6 +1021,7 @@ class TorrentManager(component.Component):
         # Torrent may need to download data after checking.
         if torrent.state in ('Checking', 'Checking Resume Data', 'Downloading'):
             torrent.is_finished = False
+            self.queued_torrents.add(torrent_id)
 
         # Only emit a state changed event if the state has actually changed
         if torrent.state != old_state: