Python implementation of a peer file transfer instance

Considering the increasing number of servers in my hand, sometimes it is necessary to deploy the same file on a large scale, for example, because it is convenient to use the Systemtap tool to locate the problem, it needs to install kernel-debuginfo This package at the same time with hundreds of servers, the original way of using a source server, Using Rsync or SCP and other file transfer method can only do a point down to distribute this file, this time issued by the speed will be slower, based on the above reasons, I wrote a BT protocol transfer files of the small tool, the actual test, transmission to 10 room, More than 70 machines to transmit a 240M of this kernel file, to all the machine, the source uses speed limit 2m/s upload rate, the test results about as long as 140s, you can complete the transmission, this efficiency is very high, if the speed is not limited to the situation faster, the following the program open source out.

The code is as follows:

#!/usr/bin/env python

Import Libtorrent as Lt
Import Sys
Import OS
Import time
From Optparse import Optionparser
Import socket
Import struct
Import Fcntl

def get_interface_ip (ifname):
s = socket.socket (socket.af_inet, socket. SOCK_DGRAM)
Return Socket.inet_ntoa (Fcntl.ioctl (S.fileno (), 0x8915, Struct.pack (' 256s ',
IFNAME[:15])) [20:24])
def ip2long (IP):
return reduce (lambda A, B: (a<<8) +b,[int (i) for I in Ip.split ('. ')])


Def get_wan_ip_address ():
interfaces = set ([' eth0 ', ' eth1 ', ' eth2 ', ' eth3 ', ' em1 ', ' em2 ', ' em3 ', ' em4 '])
IP = "
For I in interfaces:
Try
ip = get_interface_ip (i)
if (Ip2long (IP) < Ip2long (' 10.0.0.0 ') or Ip2long (IP) > Ip2long (' 10.255.255.255 ')) \
and (Ip2long (IP) < Ip2long (' 172.16.0.0 ') or Ip2long (IP) > Ip2long (' 172.33.255.255 ')) \
and (Ip2long (IP) < Ip2long (' 192.168.0.0 ') or Ip2long (IP) > Ip2long (' 192.168.255.255 ')):
return IP
Except
Pass

return IP

def make_torrent (Path, save):
FS = Lt.file_storage ()
Lt.add_files (FS, path)
If fs.num_files () = = 0:
print ' No files added '
Sys.exit (1)

input = Os.path.abspath (path)
basename = os.path.basename (path)
t = lt.create_torrent (FS, 0, 4 * 1024 * 1024)

T.add_tracker ("Http://10.0.1.5:8760/announce")
T.set_creator (' libtorrent%s '% lt.version)

Lt.set_piece_hashes (t, os.path.split (input) [0], Lambda x:sys.stderr.write ('. '))
Sys.stderr.write (' \ n ')

Save = os.path.dirname (Input)
Save = "%s/%s.torrent"% (save, basename)
F=open (Save, "WB")
F.write (Lt.bencode (T.generate ()))
F.close ()
print "The BT torrent file is store at%s"% save


def dl_status (handle):
While not (Handle.is_seed ()):
s = Handle.status ()

STATE_STR = [' Queued ', ' checking ', ' downloading metadata ', \
' Downloading ', ' finished ', ' seeding ', ' allocating ', ' checking fastresume ']
print ' \ractive_time:%d,%.2f%% complete (under:%.1f kb/s up:%.1f kb/s peers:%d, Seeds:%d)%s '% \
(S.active_time, S.progress *, s.download_rate/1000, s.upload_rate/1000, \
S.num_peers, S.num_seeds, State_str[s.state]),
Sys.stdout.flush ()

Time.sleep (1)
Def seed_status (handle, seedtime=100):
seedtime = Int (seedtime)
If Seedtime < 100:
Seedtime = 100
While Seedtime > 0:
Seedtime-= 1
s = Handle.status ()

STATE_STR = [' Queued ', ' checking ', ' downloading metadata ', \
' Downloading ', ' finished ', ' seeding ', ' allocating ', ' checking fastresume ']
print ' \rseed_time:%d,%.2f%% complete (under:%.1f kb/s up:%.1f kb/s peers:%d, Seeds:%d)%s '% \
(S.active_time, S.progress *, s.download_rate/1000, s.upload_rate/1000, \
S.num_peers, S.num_seeds, State_str[s.state]),
Sys.stdout.flush ()

Time.sleep (1)

def remove_torrents (torrent, session):
Session.remove_torrent (Torrent)

Def read_alerts (session):
Alert = Session.pop_alert ()
While alert:
#print Alert, Alert.message ()
Alert = Session.pop_alert ()

def download (torrent, path, upload_rate_limit=0, seedtime=100):
Try
Session = Lt.session ()
Session.set_alert_queue_size_limit (1024 * 1024)

STS = Lt.session_settings ()
Sts.ssl_listen = False
Sts.user_agent = "Thunder Deploy System"
Sts.tracker_completion_timeout = 5
Sts.tracker_receive_timeout = 5
Sts.stop_tracker_timeout = 5
Sts.active_downloads =-1
Sts.active_seeds =-1
Sts.active_limit =-1
Sts.auto_scrape_min_interval = 5
Sts.udp_tracker_token_expiry = 120
Sts.min_announce_interval = 1
Sts.inactivity_timeout = 60
Sts.connection_speed = 10
STS.ALLOW_MULTIPLE_CONNECTIONS_PER_IP = True
Sts.max_out_request_queue = 128
Sts.request_queue_size = 3

Sts.use_read_cache = False
Session.set_settings (STS)

Session.set_alert_mask (lt.alert.category_t.tracker_notification | lt.alert.category_t.status_notification)
Session.set_alert_mask (lt.alert.category_t.status_notification)

IPAddr = Get_wan_ip_address ()
#print ipaddr
if ipaddr = = "":
Session.listen_on (6881, 6881)
Else
Session.listen_on (6881, 6881, ipaddr)

limit = Int (upload_rate_limit)
If limit>=100:
Session.set_upload_rate_limit (limit*1024)
Session.set_local_upload_rate_limit (limit*1024)
Print Session.upload_rate_limit ()
Torrent_info = lt.torrent_info (torrent)
Add_params = {
' Save_path ': path,
' Storage_mode ': Lt.storage_mode_t.storage_mode_sparse,
' Paused ': False,
' auto_managed ': True,
' Ti ': torrent_info,
}

Handle = Session.add_torrent (add_params)

Read_alerts (session)
st = Time.time ()
Dl_status (handle)
ET = Time.time ()-St
print ' \nall file download in%.2f\nstart to seeding\n '% et
Sys.stdout.write (' \ n ')
Handle.super_seeding ()
Seed_status (handle, Seedtime)

Remove_torrents (handle, session)
Assert Len (session.get_torrents ()) = = 0

Finally
print ' Download finished '

if __name__ = = ' __main__ ':
Usage = "Usage:%prog [options] \ n \
%prog-d-F -S \ n \
or \ n \
%prog-m-P -S \ n "

Parser = Optionparser (usage=usage)
Parser.add_option ("-D", "--download", dest= "Download",
help= "start to download File", action= "Store_false", Default=true)
Parser.add_option ("-F", "--file", dest= "file",
help= "Torrent file")
Parser.add_option ("-U", "--upload", dest= "upload",
help= "Set upload rate limit, default was not limit", default=0)
Parser.add_option ("-T", "--time", dest= "time",
help= "Set seed time, default is 100s", default=100)
Parser.add_option ("-P", "--path", dest= "path",
help= "to make torrent with this path")
Parser.add_option ("-M", "--make", dest= "make",
help= "Make torrent", action= "Store_false", Default=true)
Parser.add_option ("-S", "--save", dest= "save",
help= "File save path, default is store to./", default= "./")
(options, args) = Parser.parse_args ()
#download (Sys.argv[1])
If Len (sys.argv)! = 6 and Len (SYS.ARGV)! = 4 and Len (SYS.ARGV)! = 8 and Len (SYS.ARGV)! = 10:
Parser.print_help ()
Sys.exit ()
if options.download = = False and Options.file! = "":
Download (Options.file, Options.save, Options.upload, Options.time)
elif Options.make = = False and Options.path! = "":
Make_torrent (Options.path, Options.save)

Preparation environment:
Need to install a libtorrent library on all of the OS, download the address:

Http://code.google.com/p/libtorrent/downloads/list

Remember to take the./configure–enable-python-binding when compiling, then Mak,make install, enter the binding directory, Make,make install
You can run this little tool.
Of course, large-scale deployment is not possible to use each to compile the installation of the way, as long as the compiled libtorrent.so libtorrent-rasterbar.so.7 file with bt.py This file in the same directory, and write a shell script

The code is as follows:

Lib= ' DirName
Export ld_library_path= $LD _library_path: $lib
Python bt.py-d-F- <种子文件> S- <文件保存路径> T- <做种时间> u <限制上传速度>

How to use:
First generate the seed file on the source server

The code is as follows:

Python bt.py-m-P- <要发布的文件或者文件夹> s <种子保存地址>

Publish a file
Above the source server, execute the

The code is as follows:

Python bt.py-d-F- <种子文件> S- <文件保存路径> T- <做种时间> u <限制上传速度>

One of the time to do the default setting is 100s, upload speed is not limited by default, the speed limit is the unit of KB

Below the machine, directly can

The code is as follows:

Python bt.py-d-F- <种子文件> S- <文件保存路径> t <做种时间>

As long as a machine is finished, automatically as a seed, in the process of downloading will also be uploaded, any machine can be used as the source server, of course, there is also a central tracker server, script, I built a tracker source server, Put to the 10.0.1.5 port is 8760 above, of course, you can also use OpenTracker this software to build a tracker server, modify the source code of the corresponding part, in addition to the release is a private file, the code as has been banned DHT, if you want to be more secure, build a private TR Acker server, the concrete way to build the use of search engines, find out how to build!

At present, the basic can be used, follow-up consideration is more simple, the use of magnetic links, so you can do not need to copy a seed file each, a separate command line can be published throughout the file

