[#2889] Fixes for 'Too many files open' error

* Ensure all file descriptors are closed. Using the with statement ensures
   closure and adding 'from __future__ ...' for Python 2.5 compatibility.
 * The main problem was with thousands of unclosed file desciptors from
   tracker_icons mkstemp.
 * Use a prefix 'deluge_ticon.' to identify created tracker_icon tmp files.

deluge/common.py

@@ -36,6 +36,8 @@
 
 """Common functions for various parts of Deluge to use."""
 
+from __future__ import with_statement
+
 import os
 import time
 import subprocess
@@ -177,10 +179,11 @@ def get_default_download_dir():
     if not windows_check():
         from xdg.BaseDirectory import xdg_config_home
         try:
-            for line in open(os.path.join(xdg_config_home, 'user-dirs.dirs'), 'r'):
+            with open(os.path.join(xdg_config_home, 'user-dirs.dirs'), 'r') as _file:
-                if not line.startswith('#') and line.startswith('XDG_DOWNLOAD_DIR'):
+                for line in _file:
-                    download_dir = os.path.expandvars(line.partition("=")[2].rstrip().strip('"'))
+                    if not line.startswith('#') and line.startswith('XDG_DOWNLOAD_DIR'):
-                    break
+                        download_dir = os.path.expandvars(line.partition("=")[2].rstrip().strip('"'))
+                        break
         except IOError:
             pass
 

deluge/config.py

@@ -67,6 +67,8 @@ version as this will be done internally.
 
 """
 
+from __future__ import with_statement
+
 import cPickle as pickle
 import shutil
 import os
@@ -356,7 +358,8 @@ what is currently in the config and it could not convert the value
             filename = self.__config_file
 
         try:
-            data = open(filename, "rb").read()
+            with open(filename, "rb") as _file:
+                data = _file.read()
         except IOError, e:
             log.warning("Unable to open config file %s: %s", filename, e)
             return
@@ -404,7 +407,8 @@ what is currently in the config and it could not convert the value
         # Check to see if the current config differs from the one on disk
         # We will only write a new config file if there is a difference
         try:
-            data = open(filename, "rb").read()
+            with open(filename, "rb") as _file:
+                data = _file.read()
             objects = find_json_objects(data)
             start, end = objects[0]
             version = json.loads(data[start:end])

deluge/core/authmanager.py

@@ -33,6 +33,8 @@
 #
 #
 
+from __future__ import with_statement
+
 import os
 import random
 import stat
@@ -118,7 +120,8 @@ class AuthManager(component.Component):
             f = [localclient]
         else:
             # Load the auth file into a dictionary: {username: password, ...}
-            f = open(auth_file, "r").readlines()
+            with open(auth_file, "r") as _file:
+                f = _file.readlines()
 
         for line in f:
             line = line.strip()
@@ -143,4 +146,5 @@ class AuthManager(component.Component):
             self.__auth[username.strip()] = (password.strip(), level)
 
         if "localclient" not in self.__auth:
-            open(auth_file, "a").write(self.__create_localclient_account())
+            with open(auth_file, "a") as _file:
+                _file.write(self.__create_localclient_account())

deluge/core/core.py

@@ -33,6 +33,8 @@
 #
 #
 
+from __future__ import with_statement
+
 from deluge._libtorrent import lt
 
 import os
@@ -186,8 +188,8 @@ class Core(component.Component):
     def __load_session_state(self):
         """Loads the libtorrent session state"""
         try:
-            self.session.load_state(lt.bdecode(
+            with open(deluge.configmanager.get_config_dir("session.state"), "rb") as _file:
-                open(deluge.configmanager.get_config_dir("session.state"), "rb").read()))
+                self.session.load_state(lt.bdecode(_file.read()))
         except Exception, e:
             log.warning("Failed to load lt state: %s", e)
 
@@ -661,7 +663,8 @@ class Core(component.Component):
         if add_to_session:
             options = {}
             options["download_location"] = os.path.split(path)[0]
-            self.add_torrent_file(os.path.split(target)[1], open(target, "rb").read(), options)
+            with open(target, "rb") as _file:
+                self.add_torrent_file(os.path.split(target)[1], _file.read(), options)
 
     @export
     def upload_plugin(self, filename, filedump):

deluge/core/daemon.py

@@ -32,6 +32,8 @@
 #    statement from all source files in the program, then also delete it here.
 #
 
+from __future__ import with_statement
+
 import os
 import gettext
 import locale
@@ -52,7 +54,8 @@ class Daemon(object):
         if os.path.isfile(deluge.configmanager.get_config_dir("deluged.pid")):
             # Get the PID and the port of the supposedly running daemon
             try:
-                (pid, port) = open(deluge.configmanager.get_config_dir("deluged.pid")).read().strip().split(";")
+                with open(deluge.configmanager.get_config_dir("deluged.pid")) as _file:
+                    (pid, port) = _file.read().strip().split(";")
                 pid = int(pid)
                 port = int(port)
             except ValueError:
@@ -169,8 +172,8 @@ class Daemon(object):
         if not classic:
             # Write out a pid file all the time, we use this to see if a deluged is running
             # We also include the running port number to do an additional test
-            open(deluge.configmanager.get_config_dir("deluged.pid"), "wb").write(
+            with open(deluge.configmanager.get_config_dir("deluged.pid"), "wb") as _file:
-                "%s;%s\n" % (os.getpid(), port))
+                _file.write("%s;%s\n" % (os.getpid(), port))
 
             component.start()
             try:

deluge/core/preferencesmanager.py

@@ -33,6 +33,7 @@
 #
 #
 
+from __future__ import with_statement
 
 import os.path
 import threading
@@ -298,7 +299,8 @@ class PreferencesManager(component.Component):
         if value:
             state = None
             try:
-                state = lt.bdecode(open(state_file, "rb").read())
+                with open(state_file, "rb") as _file:
+                    state = lt.bdecode(_file.read())
             except Exception, e:
                 log.warning("Unable to read DHT state file: %s", e)
 

deluge/core/rpcserver.py

@@ -35,6 +35,8 @@
 
 """RPCServer Module"""
 
+from __future__ import with_statement
+
 import sys
 import zlib
 import os
@@ -517,8 +519,10 @@ def generate_ssl_keys():
 
     # Write out files
     ssl_dir = deluge.configmanager.get_config_dir("ssl")
-    open(os.path.join(ssl_dir, "daemon.pkey"), "w").write(crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey))
+    with open(os.path.join(ssl_dir, "daemon.pkey"), "w") as _file:
-    open(os.path.join(ssl_dir, "daemon.cert"), "w").write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
+        _file.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey))
+    with open(os.path.join(ssl_dir, "daemon.cert"), "w") as _file:
+        _file.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
     # Make the files only readable by this user
     for f in ("daemon.pkey", "daemon.cert"):
         os.chmod(os.path.join(ssl_dir, f), stat.S_IREAD | stat.S_IWRITE)

deluge/core/torrent.py

@@ -34,6 +34,8 @@
 
 """Internal Torrent class"""
 
+from __future__ import with_statement
+
 import os
 import time
 from urllib import unquote
@@ -957,7 +959,8 @@ class Torrent(object):
             md = lt.bdecode(self.torrent_info.metadata())
             torrent_file = {}
             torrent_file["info"] = md
-            open(path, "wb").write(lt.bencode(torrent_file))
+            with open(path, "wb") as _file:
+                _file.write(lt.bencode(torrent_file))
         except Exception, e:
             log.warning("Unable to save torrent file: %s", e)
 

deluge/main.py

@@ -39,6 +39,8 @@
 
 """Main starting point for Deluge.  Contains the main() entry point."""
 
+from __future__ import with_statement
+
 import os
 import sys
 from optparse import OptionParser
@@ -207,7 +209,8 @@ this should be an IP address", metavar="IFACE",
     # Writes out a pidfile if necessary
     def write_pidfile():
         if options.pidfile:
-            open(options.pidfile, "wb").write("%s\n" % os.getpid())
+            with open(options.pidfile, "wb") as _file:
+                _file.write("%s\n" % os.getpid())
 
     # If the donot daemonize is set, then we just skip the forking
     if not options.donot:

deluge/maketorrent.py

@@ -33,6 +33,8 @@
 #
 #
 
+from __future__ import with_statement
+
 import sys
 import os
 from hashlib import sha1 as sha
@@ -218,11 +220,13 @@ class TorrentMetadata(object):
                     progress(len(pieces), num_pieces)
 
                 r = fd.read(piece_size)
+            fd.close()
 
             torrent["info"]["pieces"] = "".join(pieces)
 
         # Write out the torrent file
-        open(torrent_path, "wb").write(bencode(torrent))
+        with open(torrent_path, "wb") as _file:
+            _file.write(bencode(torrent))
 
     def get_data_path(self):
         """

deluge/ui/common.py

@@ -38,6 +38,8 @@ The ui common module contains methods and classes that are deemed useful for
 all the interfaces.
 """
 
+from __future__ import with_statement
+
 import os
 import sys
 import urlparse
@@ -66,7 +68,8 @@ class TorrentInfo(object):
         # Get the torrent data from the torrent file
         try:
             log.debug("Attempting to open %s.", filename)
-            self.__m_filedata = open(filename, "rb").read()
+            with open(filename, "rb") as _file:
+                self.__m_filedata = _file.read()
             self.__m_metadata = bencode.bdecode(self.__m_filedata)
         except Exception, e:
             log.warning("Unable to open %s: %s", filename, e)
@@ -409,26 +412,27 @@ def get_localhost_auth():
     """
     auth_file = deluge.configmanager.get_config_dir("auth")
     if os.path.exists(auth_file):
-        for line in open(auth_file):
+        with open(auth_file) as _file:
-            line = line.strip()
+            for line in _file:
-            if line.startswith("#") or not line:
+                line = line.strip()
-                # This is a comment or blank line
+                if line.startswith("#") or not line:
-                continue
+                    # This is a comment or blank line
+                    continue
 
-            try:
+                try:
-                lsplit = line.split(":")
+                    lsplit = line.split(":")
-            except Exception, e:
+                except Exception, e:
-                log.error("Your auth file is malformed: %s", e)
+                    log.error("Your auth file is malformed: %s", e)
-                continue
+                    continue
 
-            if len(lsplit) == 2:
+                if len(lsplit) == 2:
-                username, password = lsplit
+                    username, password = lsplit
-            elif len(lsplit) == 3:
+                elif len(lsplit) == 3:
-                username, password, level = lsplit
+                    username, password, level = lsplit
-            else:
+                else:
-                log.error("Your auth file is malformed: Incorrect number of fields!")
+                    log.error("Your auth file is malformed: Incorrect number of fields!")
-                continue
+                    continue
 
-            if username == "localclient":
+                if username == "localclient":
-                return (username, password)
+                    return (username, password)
     return ("", "")

deluge/ui/console/commands/add.py

@@ -33,6 +33,9 @@
 #    statement from all source files in the program, then also delete it here.
 #
 #
+
+from __future__ import with_statement
+
 from twisted.internet import defer
 
 from deluge.ui.console.main import BaseCommand
@@ -86,7 +89,8 @@ class Command(BaseCommand):
                     continue
                 self.console.write("{!info!}Attempting to add torrent: %s" % arg)
                 filename = os.path.split(arg)[-1]
-                filedump = base64.encodestring(open(arg, "rb").read())
+                with open(arg, "rb") as _file:
+                    filedump = base64.encodestring(_file.read())
                 deferreds.append(client.core.add_torrent_file(filename, filedump, t_options).addCallback(on_success).addErrback(on_fail))
 
         return defer.DeferredList(deferreds)

deluge/ui/gtkui/ipcinterface.py

@@ -33,6 +33,7 @@
 #
 #
 
+from __future__ import with_statement
 
 import sys
 import os
@@ -106,12 +107,14 @@ class IPCInterface(component.Component):
                 port = random.randrange(20000, 65535)
                 reactor.listenTCP(port, self.factory)
                 # Store the port number in the socket file
-                open(socket, "w").write(str(port))
+                with open(socket, "w") as _file:
+                    _file.write(str(port))
                 # We need to process any args when starting this process
                 process_args(args)
             else:
                 # Send to existing deluge process
-                port = int(open(socket, "r").readline())
+                with open(socket) as _file:
+                    port = int(_file.readline())
                 self.factory = ClientFactory()
                 self.factory.args = args
                 self.factory.protocol = IPCProtocolClient
@@ -221,4 +224,6 @@ def process_args(args):
                 component.get("AddTorrentDialog").add_from_files([path])
                 component.get("AddTorrentDialog").show(config["focus_add_dialog"])
             else:
-                client.core.add_torrent_file(os.path.split(path)[-1], base64.encodestring(open(path, "rb").read()), None)
+                with open(path, "rb") as _file:
+                    filedump = base64.encodestring(_file.read())
+                client.core.add_torrent_file(os.path.split(path)[-1], filedump, None)

deluge/ui/gtkui/preferences.py

@@ -33,6 +33,7 @@
 #
 #
 
+from __future__ import with_statement
 
 import pygtk
 pygtk.require('2.0')
@@ -958,7 +959,8 @@ class Preferences(component.Component):
 
         if not client.is_localhost():
             # We need to send this plugin to the daemon
-            filedump = base64.encodestring(open(filepath, "rb").read())
+            with open(filepath, "rb") as _file:
+                filedump = base64.encodestring(_file.read())
             client.core.upload_plugin(filename, filedump)
 
         client.core.rescan_plugins()

deluge/ui/tracker_icons.py

@@ -33,6 +33,8 @@
 #
 #
 
+from __future__ import with_statement
+
 import os
 from HTMLParser import HTMLParser, HTMLParseError
 from urlparse import urljoin, urlparse
@@ -226,7 +228,9 @@ class TrackerIcons(Component):
         if not url:
             url = self.host_to_url(host)
         log.debug("Downloading %s %s", host, url)
-        return download_file(url, mkstemp()[1], force_filename=True)
+        fd, filename = mkstemp(prefix='deluge_ticon.')
+        os.close(fd)
+        return download_file(url, filename, force_filename=True)
 
     def on_download_page_complete(self, page):
         """
@@ -355,7 +359,8 @@ class TrackerIcons(Component):
 
         if PIL_INSTALLED:
             try:
-                Image.open(icon_name)
+                with Image.open(icon_name):
+                    pass
             except IOError, e:
                 raise InvalidIconError(e)
         else:
@@ -429,14 +434,14 @@ class TrackerIcons(Component):
         """
         if icon:
             filename = icon.get_filename()
-            img = Image.open(filename)
+            with Image.open(filename) as img:
-            if img.size > (16, 16):
+                if img.size > (16, 16):
-                new_filename = filename.rpartition('.')[0]+".png"
+                    new_filename = filename.rpartition('.')[0]+".png"
-                img = img.resize((16, 16), Image.ANTIALIAS)
+                    img = img.resize((16, 16), Image.ANTIALIAS)
-                img.save(new_filename)
+                    img.save(new_filename)
-                if new_filename != filename:
+                    if new_filename != filename:
-                    os.remove(filename)
+                        os.remove(filename)
-                    icon = TrackerIcon(new_filename)
+                        icon = TrackerIcon(new_filename)
         return icon
 
     def store_icon(self, icon, host):

deluge/ui/web/json_api.py

@@ -33,6 +33,8 @@
 #
 #
 
+from __future__ import with_statement
+
 import os
 import time
 import base64
@@ -782,7 +784,8 @@ class WebApi(JSONComponent):
                 client.core.add_torrent_magnet(torrent["path"], torrent["options"])
             else:
                 filename = os.path.basename(torrent["path"])
-                fdump = base64.encodestring(open(torrent["path"], "rb").read())
+                with open(torrent["path"], "rb") as _file:
+                    fdump = base64.encodestring(_file.read())
                 log.info("Adding torrent from file `%s` with options `%r`",
                          filename, torrent["options"])
                 client.core.add_torrent_file(filename, fdump, torrent["options"])
@@ -991,7 +994,8 @@ class WebApi(JSONComponent):
             client.core.rescan_plugins()
             return True
 
-        plugin_data = base64.encodestring(open(path, "rb").read())
+        with open(path, "rb") as _file:
+            plugin_data = base64.encodestring(_file.read())
 
         def on_upload_complete(*args):
             client.core.rescan_plugins()

deluge/ui/web/server.py

@@ -234,9 +234,10 @@ class Flag(resource.Resource):
             request.setHeader("cache-control",
                               "public, must-revalidate, max-age=86400")
             request.setHeader("content-type", "image/png")
-            data = open(filename, "rb")
+            with open(filename, "rb") as _file:
+                data = _file.read()
             request.setResponseCode(http.OK)
-            return data.read()
+            return data
         else:
             request.setResponseCode(http.NOT_FOUND)
             return ""
@@ -282,7 +283,9 @@ class LookupResource(resource.Resource, component.Component):
                 log.debug("Serving path: '%s'", path)
                 mime_type = mimetypes.guess_type(path)
                 request.setHeader("content-type", mime_type[0])
-                return compress(open(path, "rb").read(), request)
+                with open(path, "rb") as _file:
+                    data = _file.read()
+                return compress(data, request)
 
         request.setResponseCode(http.NOT_FOUND)
         return "<h1>404 - Not Found</h1>"
@@ -386,19 +389,20 @@ class ScriptResource(resource.Resource, component.Component):
 
                         order_file = os.path.join(dirpath, '.order')
                         if os.path.isfile(order_file):
-                            for line in open(order_file, 'rb'):
+                            with open(order_file, 'rb') as _file:
-                                line = line.strip()
+                                for line in _file:
-                                if not line or line[0] == '#':
+                                    line = line.strip()
-                                    continue
+                                    if not line or line[0] == '#':
-                                try:
+                                        continue
-                                    pos, filename = line.split()
+                                    try:
-                                    files.pop(files.index(filename))
+                                        pos, filename = line.split()
-                                    if pos == '+':
+                                        files.pop(files.index(filename))
-                                        files.insert(0, filename)
+                                        if pos == '+':
-                                    else:
+                                            files.insert(0, filename)
-                                        files.append(filename)
+                                        else:
-                                except:
+                                            files.append(filename)
-                                    pass
+                                    except:
+                                        pass
 
                         dirpath = dirpath[len(filepath)+1:]
                         if dirpath:
@@ -439,7 +443,9 @@ class ScriptResource(resource.Resource, component.Component):
                 log.debug("Serving path: '%s'", path)
                 mime_type = mimetypes.guess_type(path)
                 request.setHeader("content-type", mime_type[0])
-                return compress(open(path, "rb").read(), request)
+                with open(path, "rb") as _file:
+                    data = _file.read()
+                return compress(data, request)
 
         request.setResponseCode(http.NOT_FOUND)
         return "<h1>404 - Not Found</h1>"