mirrors/deluge
0
0
Fork 0
mirror of https://git.deluge-torrent.org/deluge synced 2025-08-09 01:48:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[Dependency] Remove bundled rencode

Browse source
This commit is contained in:
Calum Lind 2018-11-01 20:53:46 +00:00
commit e7127637cf
9 changed files with 8 additions and 532 deletions
Download patch file Download diff file Expand all files Collapse all files

2
DEPENDS.md
View file

@ -20,6 +20,7 @@ All modules will require the [common](#common) section dependencies.
- [Twisted] _>= 16.6_ - Use `TLS` extras for `service_identity` and `idna`. - [Twisted] _>= 16.6_ - Use `TLS` extras for `service_identity` and `idna`.
- [OpenSSL] _>= 1.0.1_ - [OpenSSL] _>= 1.0.1_
- [pyOpenSSL] - [pyOpenSSL]
- [rencode] _>= 1.0.2_ - Encoding library.
- [PyXDG] - Access freedesktop.org standards for \*nix. - [PyXDG] - Access freedesktop.org standards for \*nix.
- [xdg-utils] - Provides xdg-open for \*nix. - [xdg-utils] - Provides xdg-open for \*nix.
- [six] - [six]
@ -27,7 +28,6 @@ All modules will require the [common](#common) section dependencies.
- [chardet] - Optional: Encoding detection. - [chardet] - Optional: Encoding detection.
- [setproctitle] - Optional: Renaming processes. - [setproctitle] - Optional: Renaming processes.
- [Pillow] - Optional: Support for resizing tracker icons. - [Pillow] - Optional: Support for resizing tracker icons.
- [rencode] _>= 1.0.2_ - Optional: Encoding library (Python port is bundled).
- [dbus-python] - Optional: Show item location in filemanager. - [dbus-python] - Optional: Show item location in filemanager.
#### Windows OS #### Windows OS

519
deluge/rencode.py
View file

@ -1,519 +0,0 @@
# Original bencode module by Petru Paler, et al.
#
# Modifications by Connelly Barnes:
#
# - Added support for floats (sent as 32-bit or 64-bit in network
# order), bools, None.
# - Allowed dict keys to be of any serializable type.
# - Lists/tuples are always decoded as tuples (thus, tuples can be
# used as dict keys).
# - Embedded extra information in the 'typecodes' to save some space.
# - Added a restriction on integer length, so that malicious hosts
# cannot pass us large integers which take a long time to decode.
#
# Licensed by Bram Cohen under the "MIT license":
#
# "Copyright (C) 2001-2002 Bram Cohen
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# The Software is provided "AS IS", without warranty of any kind,
# express or implied, including but not limited to the warranties of
# merchantability, fitness for a particular purpose and
# noninfringement. In no event shall the authors or copyright holders
# be liable for any claim, damages or other liability, whether in an
# action of contract, tort or otherwise, arising from, out of or in
# connection with the Software or the use or other dealings in the
# Software."
#
# (The rencode module is licensed under the above license as well).
#
"""
rencode -- Web safe object pickling/unpickling.
Public domain, Connelly Barnes 2006-2007.
The rencode module is a modified version of bencode from the
BitTorrent project. For complex, heterogeneous data structures with
many small elements, r-encodings take up significantly less space than
b-encodings:
>>> len(rencode.dumps({'a':0, 'b':[1,2], 'c':99}))
13
>>> len(bencode.bencode({'a':0, 'b':[1,2], 'c':99}))
26
The rencode format is not standardized, and may change with different
rencode module versions, so you should check that you are using the
same rencode version throughout your project.
"""
import struct
import sys
from threading import Lock
try:
from future_builtins import zip
except ImportError:
# Ignore on Py3.
pass
__version__ = ('Python', 1, 0, 4)
__all__ = ['dumps', 'loads']
py3 = sys.version_info[0] >= 3
if py3:
long = int # noqa: A001, pylint: disable=redefined-builtin
unicode = str # noqa: A001, pylint: disable=redefined-builtin
def int2byte(c):
return bytes([c])
else:
def int2byte(c):
return chr(c)
# Default number of bits for serialized floats, either 32 or 64 (also a parameter for dumps()).
DEFAULT_FLOAT_BITS = 32
# Maximum length of integer when written as base 10 string.
MAX_INT_LENGTH = 64
# The bencode 'typecodes' such as i, d, etc have been extended and
# relocated on the base-256 character set.
CHR_LIST = int2byte(59)
CHR_DICT = int2byte(60)
CHR_INT = int2byte(61)
CHR_INT1 = int2byte(62)
CHR_INT2 = int2byte(63)
CHR_INT4 = int2byte(64)
CHR_INT8 = int2byte(65)
CHR_FLOAT32 = int2byte(66)
CHR_FLOAT64 = int2byte(44)
CHR_TRUE = int2byte(67)
CHR_FALSE = int2byte(68)
CHR_NONE = int2byte(69)
CHR_TERM = int2byte(127)
# Positive integers with value embedded in typecode.
INT_POS_FIXED_START = 0
INT_POS_FIXED_COUNT = 44
# Dictionaries with length embedded in typecode.
DICT_FIXED_START = 102
DICT_FIXED_COUNT = 25
# Negative integers with value embedded in typecode.
INT_NEG_FIXED_START = 70
INT_NEG_FIXED_COUNT = 32
# Strings with length embedded in typecode.
STR_FIXED_START = 128
STR_FIXED_COUNT = 64
# Lists with length embedded in typecode.
LIST_FIXED_START = STR_FIXED_START + STR_FIXED_COUNT
LIST_FIXED_COUNT = 64
# Whether strings should be decoded when loading
_decode_utf8 = False
def decode_int(x, f):
f += 1
newf = x.index(CHR_TERM, f)
if newf - f >= MAX_INT_LENGTH:
raise ValueError('overflow')
try:
n = int(x[f:newf])
except (OverflowError, ValueError):
n = long(x[f:newf])
if x[f : f + 1] == '-':
if x[f + 1 : f + 2] == '0':
raise ValueError
elif x[f : f + 1] == '0' and newf != f + 1:
raise ValueError
return (n, newf + 1)
def decode_intb(x, f):
f += 1
return (struct.unpack('!b', x[f : f + 1])[0], f + 1)
def decode_inth(x, f):
f += 1
return (struct.unpack('!h', x[f : f + 2])[0], f + 2)
def decode_intl(x, f):
f += 1
return (struct.unpack('!l', x[f : f + 4])[0], f + 4)
def decode_intq(x, f):
f += 1
return (struct.unpack('!q', x[f : f + 8])[0], f + 8)
def decode_float32(x, f):
f += 1
n = struct.unpack('!f', x[f : f + 4])[0]
return (n, f + 4)
def decode_float64(x, f):
f += 1
n = struct.unpack('!d', x[f : f + 8])[0]
return (n, f + 8)
def decode_string(x, f):
colon = x.index(b':', f)
try:
n = int(x[f:colon])
except (OverflowError, ValueError):
n = long(x[f:colon])
if x[f] == '0' and colon != f + 1:
raise ValueError
colon += 1
s = x[colon : colon + n]
if _decode_utf8:
s = s.decode('utf8')
return (s, colon + n)
def decode_list(x, f):
r, f = [], f + 1
while x[f : f + 1] != CHR_TERM:
v, f = decode_func[x[f : f + 1]](x, f)
r.append(v)
return (tuple(r), f + 1)
def decode_dict(x, f):
r, f = {}, f + 1
while x[f : f + 1] != CHR_TERM:
k, f = decode_func[x[f : f + 1]](x, f)
r[k], f = decode_func[x[f : f + 1]](x, f)
return (r, f + 1)
def decode_true(x, f):
return (True, f + 1)
def decode_false(x, f):
return (False, f + 1)
def decode_none(x, f):
return (None, f + 1)
decode_func = {}
decode_func[b'0'] = decode_string
decode_func[b'1'] = decode_string
decode_func[b'2'] = decode_string
decode_func[b'3'] = decode_string
decode_func[b'4'] = decode_string
decode_func[b'5'] = decode_string
decode_func[b'6'] = decode_string
decode_func[b'7'] = decode_string
decode_func[b'8'] = decode_string
decode_func[b'9'] = decode_string
decode_func[CHR_LIST] = decode_list
decode_func[CHR_DICT] = decode_dict
decode_func[CHR_INT] = decode_int
decode_func[CHR_INT1] = decode_intb
decode_func[CHR_INT2] = decode_inth
decode_func[CHR_INT4] = decode_intl
decode_func[CHR_INT8] = decode_intq
decode_func[CHR_FLOAT32] = decode_float32
decode_func[CHR_FLOAT64] = decode_float64
decode_func[CHR_TRUE] = decode_true
decode_func[CHR_FALSE] = decode_false
decode_func[CHR_NONE] = decode_none
def make_fixed_length_string_decoders():
def make_decoder(slen):
def f(x, f):
s = x[f + 1 : f + 1 + slen]
if _decode_utf8:
s = s.decode('utf8')
return (s, f + 1 + slen)
return f
for i in range(STR_FIXED_COUNT):
decode_func[int2byte(STR_FIXED_START + i)] = make_decoder(i)
make_fixed_length_string_decoders()
def make_fixed_length_list_decoders():
def make_decoder(slen):
def f(x, f):
r, f = [], f + 1
for _ in range(slen):
v, f = decode_func[x[f : f + 1]](x, f)
r.append(v)
return (tuple(r), f)
return f
for i in range(LIST_FIXED_COUNT):
decode_func[int2byte(LIST_FIXED_START + i)] = make_decoder(i)
make_fixed_length_list_decoders()
def make_fixed_length_int_decoders():
def make_decoder(j):
def f(x, f):
return (j, f + 1)
return f
for i in range(INT_POS_FIXED_COUNT):
decode_func[int2byte(INT_POS_FIXED_START + i)] = make_decoder(i)
for i in range(INT_NEG_FIXED_COUNT):
decode_func[int2byte(INT_NEG_FIXED_START + i)] = make_decoder(-1 - i)
make_fixed_length_int_decoders()
def make_fixed_length_dict_decoders():
def make_decoder(slen):
def f(x, f):
r, f = {}, f + 1
for _ in range(slen):
k, f = decode_func[x[f : f + 1]](x, f)
r[k], f = decode_func[x[f : f + 1]](x, f)
return (r, f)
return f
for i in range(DICT_FIXED_COUNT):
decode_func[int2byte(DICT_FIXED_START + i)] = make_decoder(i)
make_fixed_length_dict_decoders()
def loads(x, decode_utf8=False):
global _decode_utf8
_decode_utf8 = decode_utf8
try:
r, f = decode_func[x[0:1]](x, 0)
except (IndexError, KeyError):
raise ValueError
if f != len(x):
raise ValueError
return r
def encode_int(x, r):
if 0 <= x < INT_POS_FIXED_COUNT:
r.append(int2byte(INT_POS_FIXED_START + x))
elif -INT_NEG_FIXED_COUNT <= x < 0:
r.append(int2byte(INT_NEG_FIXED_START - 1 - x))
elif -128 <= x < 128:
r.extend((CHR_INT1, struct.pack('!b', x)))
elif -32768 <= x < 32768:
r.extend((CHR_INT2, struct.pack('!h', x)))
elif -2147483648 <= x < 2147483648:
r.extend((CHR_INT4, struct.pack('!l', x)))
elif -9223372036854775808 <= x < 9223372036854775808:
r.extend((CHR_INT8, struct.pack('!q', x)))
else:
s = str(x)
if py3:
s = bytes(s, 'ascii')
if len(s) >= MAX_INT_LENGTH:
raise ValueError('overflow')
r.extend((CHR_INT, s, CHR_TERM))
def encode_float32(x, r):
r.extend((CHR_FLOAT32, struct.pack('!f', x)))
def encode_float64(x, r):
r.extend((CHR_FLOAT64, struct.pack('!d', x)))
def encode_bool(x, r):
r.append({False: CHR_FALSE, True: CHR_TRUE}[bool(x)])
def encode_none(x, r):
r.append(CHR_NONE)
def encode_string(x, r):
if len(x) < STR_FIXED_COUNT:
r.extend((int2byte(STR_FIXED_START + len(x)), x))
else:
s = str(len(x))
if py3:
s = bytes(s, 'ascii')
r.extend((s, b':', x))
def encode_unicode(x, r):
encode_string(x.encode('utf8'), r)
def encode_list(x, r):
if len(x) < LIST_FIXED_COUNT:
r.append(int2byte(LIST_FIXED_START + len(x)))
for i in x:
encode_func[type(i)](i, r)
else:
r.append(CHR_LIST)
for i in x:
encode_func[type(i)](i, r)
r.append(CHR_TERM)
def encode_dict(x, r):
if len(x) < DICT_FIXED_COUNT:
r.append(int2byte(DICT_FIXED_START + len(x)))
for k, v in x.items():
encode_func[type(k)](k, r)
encode_func[type(v)](v, r)
else:
r.append(CHR_DICT)
for k, v in x.items():
encode_func[type(k)](k, r)
encode_func[type(v)](v, r)
r.append(CHR_TERM)
encode_func = {}
encode_func[int] = encode_int
encode_func[long] = encode_int
encode_func[bytes] = encode_string
encode_func[list] = encode_list
encode_func[tuple] = encode_list
encode_func[dict] = encode_dict
encode_func[type(None)] = encode_none
encode_func[unicode] = encode_unicode
encode_func[bool] = encode_bool
lock = Lock()
def dumps(x, float_bits=DEFAULT_FLOAT_BITS):
"""
Dump data structure to str.
Here float_bits is either 32 or 64.
"""
with lock:
if float_bits == 32:
encode_func[float] = encode_float32
elif float_bits == 64:
encode_func[float] = encode_float64
else:
raise ValueError('Float bits (%d) is not 32 or 64' % float_bits)
r = []
encode_func[type(x)](x, r)
return b''.join(r)
def test():
f1 = struct.unpack('!f', struct.pack('!f', 25.5))[0]
f2 = struct.unpack('!f', struct.pack('!f', 29.3))[0]
f3 = struct.unpack('!f', struct.pack('!f', -0.6))[0]
ld = (
(
{b'a': 15, b'bb': f1, b'ccc': f2, b'': (f3, (), False, True, b'')},
(b'a', 10 ** 20),
tuple(range(-100000, 100000)),
b'b' * 31,
b'b' * 62,
b'b' * 64,
2 ** 30,
2 ** 33,
2 ** 62,
2 ** 64,
2 ** 30,
2 ** 33,
2 ** 62,
2 ** 64,
False,
False,
True,
-1,
2,
0,
),
)
assert loads(dumps(ld)) == ld
d = dict(zip(range(-100000, 100000), range(-100000, 100000)))
d.update(
{b'a': 20, 20: 40, 40: 41, f1: f2, f2: f3, f3: False, False: True, True: False}
)
ld = (d, {}, {5: 6}, {7: 7, True: 8}, {9: 10, 22: 39, 49: 50, 44: b''})
assert loads(dumps(ld)) == ld
ld = (
b'',
b'a' * 10,
b'a' * 100,
b'a' * 1000,
b'a' * 10000,
b'a' * 100000,
b'a' * 1000000,
b'a' * 10000000,
)
assert loads(dumps(ld)) == ld
ld = tuple(dict(zip(range(n), range(n))) for n in range(100)) + (b'b',)
assert loads(dumps(ld)) == ld
ld = tuple(dict(zip(range(n), range(-n, 0))) for n in range(100)) + (b'b',)
assert loads(dumps(ld)) == ld
ld = tuple(tuple(range(n)) for n in range(100)) + (b'b',)
assert loads(dumps(ld)) == ld
ld = tuple(b'a' * n for n in range(1000)) + (b'b',)
assert loads(dumps(ld)) == ld
ld = tuple(b'a' * n for n in range(1000)) + (None, True, None)
assert loads(dumps(ld)) == ld
assert loads(dumps(None)) is None
assert loads(dumps({None: None})) == {None: None}
assert 1e-10 < abs(loads(dumps(1.1)) - 1.1) < 1e-6
assert 1e-10 < abs(loads(dumps(1.1, 32)) - 1.1) < 1e-6
assert abs(loads(dumps(1.1, 64)) - 1.1) < 1e-12
assert loads(dumps('Hello World!!'), decode_utf8=True)
try:
import psyco
psyco.bind(dumps)
psyco.bind(loads)
except ImportError:
pass
if __name__ == '__main__':
test()

2
deluge/tests/test_transfer.py
View file

@ -11,10 +11,10 @@ from __future__ import print_function, unicode_literals
import base64 import base64
import rencode
from twisted.trial import unittest from twisted.trial import unittest
import deluge.log import deluge.log
import deluge.rencode as rencode
from deluge.transfer import DelugeTransferProtocol from deluge.transfer import DelugeTransferProtocol
deluge.log.setup_logger('none') deluge.log.setup_logger('none')

6
deluge/transfer.py
View file

@ -13,13 +13,9 @@ import logging
import struct import struct
import zlib import zlib
import rencode
from twisted.internet.protocol import Protocol from twisted.internet.protocol import Protocol
try:
import rencode # pylint: disable=useless-suppression,relative-import
except ImportError:
import deluge.rencode as rencode
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
MESSAGE_HEADER_SIZE = 5 MESSAGE_HEADER_SIZE = 5

6
deluge/ui/gtk3/ipcinterface.py
View file

@ -16,6 +16,7 @@ from base64 import b64encode
from glob import glob from glob import glob
from tempfile import mkstemp from tempfile import mkstemp
import rencode
import twisted.internet.error import twisted.internet.error
from twisted.internet import reactor from twisted.internet import reactor
from twisted.internet.protocol import ClientFactory, Factory, Protocol, connectionDone from twisted.internet.protocol import ClientFactory, Factory, Protocol, connectionDone
@ -25,11 +26,6 @@ from deluge.common import decode_bytes, is_magnet, is_url, windows_check
from deluge.configmanager import ConfigManager, get_config_dir from deluge.configmanager import ConfigManager, get_config_dir
from deluge.ui.client import client from deluge.ui.client import client
try:
import rencode
except ImportError:
import deluge.rencode as rencode # pylint: disable=ungrouped-imports
try: try:
from urllib.parse import urlparse from urllib.parse import urlparse
from urllib.request import url2pathname from urllib.request import url2pathname

1
docs/source/conf.py
View file

@ -91,6 +91,7 @@ for mod_name in MOCK_MODULES:
autodoc_mock_imports = [ autodoc_mock_imports = [
'twisted', 'twisted',
'rencode',
'OpenSSL', 'OpenSSL',
'PIL', 'PIL',
'libtorrent', 'libtorrent',

1
requirements.txt
View file

@ -1,4 +1,5 @@
twisted[tls]>=16.6 twisted[tls]>=16.6
rencode
pyopenssl pyopenssl
pyxdg pyxdg
pillow pillow

2
setup.cfg
View file

@ -23,7 +23,7 @@ known_third_party =
cairo, gi, cairo, gi,
# Ignore other module dependencies for pre-commit isort. # Ignore other module dependencies for pre-commit isort.
twisted, OpenSSL, pytest, recommonmark, chardet, pkg_resources, zope, mock, twisted, OpenSSL, pytest, recommonmark, chardet, pkg_resources, zope, mock,
sphinx sphinx, rencode
known_first_party = msgfmt, deluge known_first_party = msgfmt, deluge
order_by_type = true order_by_type = true
not_skip = __init__.py not_skip = __init__.py

1
setup.py
View file

@ -543,6 +543,7 @@ install_requires = [
# Add pyasn1 for setuptools workaround: # Add pyasn1 for setuptools workaround:
# https://github.com/pypa/setuptools/issues/1510 # https://github.com/pypa/setuptools/issues/1510
'pyasn1', 'pyasn1',
'rencode',
'pyopenssl', 'pyopenssl',
'pyxdg', 'pyxdg',
'pillow', 'pillow',