Changeset 290 for indytube/trunk

Timestamp:

11/26/07 10:56:05 (14 months ago)

Author:

andy

Message:

More work on the bittorrent indytube server

We have a perspective broker immplementation working, which can asyncrolously download, transcode and make a torrent file now.
We dont have an integrated tracker yet.

Location:

indytube/trunk/indytube/bittorrent

Files:

• README (modified) (1 diff)
• client.py (modified) (2 diffs)
• server.py (modified) (8 diffs)

indytube/trunk/indytube/bittorrent/README

@@ -1 +1 @@
 This is the implementation using perspective broker from twisted 
 
-
-Run the server , like so , one directory up:
-$python -ny bittorrent/server.py
+Run the server using Twistd server, like so , one directory up:
+$twistd -ny bittorrent/server.py
 
 then you can test with the main() method , in the client , like so:
 $python bittorrent/client.py
 
+
+The server needs to be running continously in normal operation.
+
+Plumi/Plone (one possible client) will connect when it needs file downloading and transcoding done.
+

indytube/trunk/indytube/bittorrent/client.py

@@ -20 +20 @@
 from twisted.spread import pb
 
-class IndyTubeClient:
+class IndyTubePBClient:
         hostname = 'localhost'
         portnumber = 9119
@@ -79 +79 @@
 def main():
         # test a simple connection to the server, on the localhost
-        client = IndyTubeClient('localhost',9119)
-        client.setupOperation('http://localhost/download/video','filename.ogg')
+        client = IndyTubePBClient('localhost',9119)
+        client.setupOperation('http://www.engagemedia.org/Members/jamyoung/videos/cold-world.mov/download','cold-world.mov')
         client.connectToServer()
         client.runReactor()

indytube/trunk/indytube/bittorrent/server.py

@@ -7 +7 @@
 #
 # This uses BitTorrent 5.0.8 to implement the bittorrent functionality
+# see http://download.bittorrent.com/dl/
 
 # Copyright Andy Nicholson 2007
@@ -31 +32 @@
 from thread import get_ident,allocate_lock
 from operator import mod
+import xmlrpclib
 
 #bittorrent
@@ -55 +57 @@
 
 #standard location for configuration file
-from indytube import CONF_FILE_DEFAULT
+from config import CONF_FILE_DEFAULT
+from indytube import IndyTubeTranscoder
 
 ## from launchmany-console
@@ -76 +79 @@
 ## end from launchmany-console
 
-class IndyTubeServer(pb.Root):
-    ''' IndyTubeServer is a perspective broker service that implements file downloading,  
+class IndyTubePBServer(pb.Root):
+    ''' IndyTubePBServer is a Twisted Perspective Broker service that implements media-file downloading,  
         mencoder and ffmpeg2theora transcoding, bittorrent meta-file creation
         and bittorrent seeding and tracking '''
+
+    #points to the filepath of the latest incoming media file downloaded
+    _incoming_fp = ''
+    #point to the latest incoming media file download URL
+    _bootstrapURL = ''
+    # the filename of the media file downloaded
+    _filename = ''
+    # torrent filename
+    _torrent_filepath = ''
+    # the logger object
+    logger = None
 
     def parse_config(self,conf_file):
@@ -87 +101 @@
 
         self.pb_protocolPort = config.get('pb-server', 'PROTOCOL_PORT')
-        self.pb_torrentDir = config.get('pb-server', 'TORRENT_DIR')
+        self.pb_torrentDir = unicode(config.get('pb-server', 'TORRENT_DIR'))
         self.pb_downloadDir = config.get('pb-server', 'DOWNLOAD_DIR')
         self.pb_serverName = config.get('pb-server', 'SERVER_NAME')
         self.pb_trackerUrl = config.get('pb-server', 'TRACKER_URL')
 
-        print "I have %s %s %s %s %s " % ( self.pb_protocolPort , self.pb_torrentDir , self.pb_downloadDir, self.pb_serverName, self.pb_trackerUrl )
+        # XXX check that these directories exist!
+
+        self.logger.info("I have %s %s %s %s %s " % ( self.pb_protocolPort , self.pb_torrentDir , self.pb_downloadDir, self.pb_serverName, self.pb_trackerUrl ))
 
     def __init__(self):
+        injectLogger(use_syslog = False, capture_output = True, verbose = True,
+                 log_level = logging.DEBUG, log_twisted = False )
+        logging.getLogger('').removeHandler(console)  # remove handler installed by BitTorrent.__init__.
+        self.logger = logging.getLogger("")
+
+        #XXX torrent seeder/tracker
         #start the torrent seeder, in a separate thread!
-        threads.deferToThread(self.initaliseLaunchManyCore)
+        #threads.deferToThread(self.initaliseLaunchManyCore)
+
+    def _doOperations(self):
+        ''' private function - just download the media at bootstrapULR into file filename'''
+        # if each chain succeeds, then go to next step, else go to error handler
+        #
+        # the chain is:
+        #1. Download the media file to a local file. 
+        #2. Call indytube to transcode it.
+        #3. Bittorrent - create torrent file of original video file.
+        #       -- Seeding and tracking - happens automatically when the xyz.torrent file is created
+        #4. Call back to Plumi/Plone to make content objects relating to these new files. 
+        #       -- create a object representing any transcoded versions of the file
+        #               just send URL links to these files
+        #       -- create a object representing the torrent file of the original video
+        #               send the actual torrent file info
+
+        self.logger.info("downloading media file from url %s, doing encoding on %s " % (self._bootstrapURL,self._filename))
+        self._incoming_fp = self.pb_downloadDir + '/' + self._filename
+        self._torrent_filepath = self.pb_torrentDir + '/' + self._filename + ".torrent"
+        filename,headers = urllib.urlretrieve(self._bootstrapURL, self._incoming_fp)
+        #we are now free to process the file .. we have downloaded it all.
+        # downloading complete!
+        self.logger.info("downloading complete!")
+        # carrying on the chain of operations..
+        self._doTranscoding()
+        self._doBittorrentOperation()
+        self._doPloneObjectCreation()
+
+    def _doTranscoding(self):
+        ''' do the transcoding via Indytube itself '''
+        # TODO - via XMLRPC ??
+
+        self.logger.info("doing transcoding of video file")     
+        # just using the python library directly
+        transcoder = IndyTubeTranscoder()
+        transcoder.parse_config(CONF_FILE_DEFAULT)
+        transcoder.attempt_transcode_file(self._incoming_fp)
+
+    def _doBittorrentOperation(self):
+        ''' Make the torrent file from the original video file '''
+        self.logger.info("Starting bittorrent operations using tracker url %s" % self.pb_trackerUrl)
+        incoming_comment = "Torrent file made by Indytube"
+        make_meta_files( self.pb_trackerUrl, [self._incoming_fp], piece_len_pow2=18, comment=incoming_comment, target = self._torrent_filepath)
+        
+    def _doPloneObjectCreation(self):
+        #  Construct a Plone based object to represent the torrent file
+        #  Construct a Plone based object to represent the transcoded files
+        self.logger.info("doing plone object creation")
+        #server_url = 'http://username:password@localhost:8080/plumitest/Members/andy/videos/test123.mov/publishXML'
+        #server = xmlrpclib.Server(server_url)
+        #server.publishObject()
+
+    def _errorinRemoteOperation(self,foo):
+        ''' error was caught during the remote operation '''
+        self.logger.info("Got an error 8-( %s " %foo)
+        return False
 
     def remote_RemoteIndyTubeOperation(self,indyInfo):
+        ''' The remote method exposed via the Perspective Broker. Clients call this method to 
+        have their media file downloaded, transcoded, and processed for bittorrent tracking and seeding '''
+
         #parse args
-        bootstrapURL=indyInfo['bootstrapURL']
-        filename=indyInfo['filename']
-
-        # XXX make all these operations run in a separate thread.
-
-        logging.getLogger(self.pb_serverName).info("downloading media file from url %s, doing encoding on %s " % (bootstrapURL,filename))
-        #1. download it 
-        incoming_fp = self.pb_downloadDir + '/' + filename
-        urllib.urlretrieve(bootstrapURL, incoming_fp)
-        
-        #2. call indytube to transcode it!!
-        # TODO
-        # transcoder = IndyTubeTranscoder()
-        # transcoder.attempt_transcode_file(incoming_fp)
-
-        #3. Bittorrent - create torrent file of original video file
-        torrent_filepath = self.pb_torrentDir + '/' + filename + ".torrent"
-        incoming_comment = "Torrent file made by Indytube"
-        make_meta_files( self.pb_trackerUrl, [incoming_fp], piece_len_pow2=18, comment=incoming_comment, target = torrent_filepath)
-
-        #4. Seeding and tracking!
-        # TODO
-
-        # everything scheduled OK - return True
+        self._bootstrapURL=indyInfo['bootstrapURL']
+        self._filename=indyInfo['filename']
+
+        self.logger.info("started - %s %s " % (self._bootstrapURL,self._filename))
+        deferred_download = threads.deferToThread(self._doOperations)
+        deferred_download.addErrback(self._errorinRemoteOperation)
+
+        #deferred_download.addCallbacks(self._do_Transcoding, self._errorinRemoteOperation)
+        #deferred_download.addCallbacks(self._doBittorrentOperation, self._errorinRemoteOperation)
+
+        # always returns True
         return True
 
@@ -132 +200 @@
             config, args = configfile.parse_configuration_and_args(defaults, uiname, [], 0, 1)
             #config, args = configfile.parse_configuration_and_args(defaults, uiname, sys.argv[1:], 0, 1)
-            config['torrent_dir'] = self.torrentDir
+            config['torrent_dir'] = self.pb_torrentDir
             config['parse_dir_interval'] = 20 #make the dir scan happen @20 seconds, not default of 60
             config['save_in'] = u""
@@ -142 +210 @@
         d = HeadlessDisplayer()
         config = Preferences().initWithDict(config)
-        injectLogger(use_syslog = False, capture_output = True, verbose = True,
-                 log_level = logging.DEBUG, log_twisted = False )
-        logging.getLogger('').removeHandler(console)  # remove handler installed by BitTorrent.__init__.
         #This starts up a RawServer, ie it will block until we are done in there...
         LaunchMany(config, d.display, 'launchmany-console')
@@ -150 +215 @@
 
 # Standard twisted application Boilerplate
-indytube_server = IndyTubeServer()
+indytube_server = IndyTubePBServer()
 #parse our config
 if not (os.path.exists(CONF_FILE_DEFAULT)):