Changeset 922

Timestamp:

07/03/09 10:03:30 (8 months ago)

Author:

andy

Message:

transcoder queue runs properly now

removed HTML generation - out of place here

added properties on transcoder queue for high/low quality versions of FLV, OGV conversion

Location:

plumimediaqueue/trunk/src/plumimediaqueue

Files:

• events/events.py (modified) (1 diff)
• events/interfaces.py (modified) (1 diff)
• interfaces.py (modified) (2 diffs)
• models.py (modified) (2 diffs)
• queues/bittorrent.py (modified) (1 diff)
• queues/download.py (modified) (3 diffs)
• queues/transcoding.py (modified) (8 diffs)

plumimediaqueue/trunk/src/plumimediaqueue/events/events.py

@@ -12 +12 @@
 
 class JobReset(grok.ObjectModifiedEvent):
-    implements(job.IJobResetedEvent)
-
-
+    implements(job.IJobResetEvent)
 
 class JobDeleted(ObjectEvent):

plumimediaqueue/trunk/src/plumimediaqueue/events/interfaces.py

@@ -10 +10 @@
     """
 
-class IJobResetedEvent(grok.IObjectModifiedEvent):
+class IJobResetEvent(grok.IObjectModifiedEvent):
     """A job has been restarted.
     """

plumimediaqueue/trunk/src/plumimediaqueue/interfaces.py

@@ -51 +51 @@
         the add will return false, unless 'replace' is True.
         In that case, existing task with the same uuid are overwritten.
-        If added to the queue, calls notify(job,IJobQueuedEvent), which should put the 
-        jobItem into STARTING state.
-        """
+           """
 
     def get_job(self, uuid):
@@ -65 +63 @@
 
     def reset_job(self,uuid):
-        """Sets the job, matched by uuid, to STARTING state"""
+        """Sets the job, matched by uuid, to STARTING state
+        """
 
     def start_job(self,uuid):

plumimediaqueue/trunk/src/plumimediaqueue/models.py

@@ -62 +62 @@
             self.id = uuid
 
+    def copy(self, other_obj):
+        self.video_download_url = other_obj.video_download_url
+        self.file_path = other_obj.file_path
+        self.processed_filepaths = other_obj.processed_filepaths
+
 class Queue(grok.Container):
     grok.implements(IQueue)
@@ -97 +102 @@
         job = self.get_job(uuid)
         job.set_status(status.STARTING)
+        evnt = events.JobReset(job)
+        evnt.newParent = self
+        notify(evnt)
+
         return True
 

plumimediaqueue/trunk/src/plumimediaqueue/queues/bittorrent.py

@@ -32 +32 @@
     if IBittorrentQueue.providedBy(q) and obj.starting():
         logging.log(logging.INFO, "New obj added: %s , with event %s at newParent %s " % (obj,event,event.newParent))
+
+        #XXX make torrent file 
+        #XXX seed
+        #XXX announce to tracker

plumimediaqueue/trunk/src/plumimediaqueue/queues/download.py

@@ -46 +46 @@
         url_parts = urlparse(download_url)
         rel_fp = url_parts[1]+url_parts[2]
-        incoming_filename = event.newParent.download_dir + '/' + rel_fp
+        incoming_filename = q.download_dir + '/' + rel_fp
         logging.debug("url parts are %s %s %s %s %s %s" % (url_parts[0],url_parts[1],url_parts[2],url_parts[3],url_parts[4],url_parts[5]))
         logging.info("downloading media file from url %s, into location %s " % (download_url,incoming_filename))
@@ -61 +61 @@
             #tell the queue we failed.
             q.fail_job(obj.uuid)
+            #nothing added to transcoding or bittorrent queues
             return
+
+        #XXX
+        #do an MD5 checksum on the file, and store it on the object
+
 
         logging.info("finished downloading")
@@ -70 +75 @@
         #Add a jobitem for the transcoding queue, and bittorrent queues.
         t_nm=NetworkMediaItem()
-        t_nm.file_path = incoming_filename
+        t_nm.copy(obj)
         bt_nm=NetworkMediaItem()
-        bt_nm.file_path = incoming_filename
+        bt_nm.copy(obj)
+        #set the both reset, into the STARTING state.
+
         q.__parent__['transcoding'].add_job(t_nm)
+
         q.__parent__['bittorrent'].add_job(bt_nm)
+

plumimediaqueue/trunk/src/plumimediaqueue/queues/transcoding.py

@@ -4 +4 @@
 
 from plumimediaqueue.models import Queue, NetworkMediaItem
-
-import logging
-import os
+from plumimediaqueue.events.interfaces import IJobResetEvent
+
+import logging , os , traceback , time, subprocess
 
 class ITranscodingQueue(IQueue):
@@ -14 +14 @@
                     )
 
+    mencoder_installed_location = Text(
+                            title = u"MEncoder", 
+                            description=u"The file path to a MEncoder executable"
+                            )
+
+    ffmpeg2theora_installed_location = Text(
+                            title = u"ffmpeg2theora", 
+                            description=u"The file path to a ffmpeg2theora executable"
+                        ) 
+
+    flvtool_installed_location = Text(
+                            title = u"flvtool2", 
+                            description=u"The file path to a flvtool2 executable"
+                        ) 
+
     mencoder_flv_options = Text(
                             title = u"MEncoder command line to produce FLV files.", 
                             description=u"The set of options to pass to MEncoder when transcoding"
                             )
+
+    ffmpeg2theora_options = Text(
+                            title = u"ffmpeg2theora command line to produce OGV files.", 
+                            description=u"The set of options to pass to ffmpeg2theora when transcoding"
+                            )
+
+
+    mencoder_flv_highquality_options = Text(
+                            title = u"MEncoder command line to produce high quality FLV files.", 
+                            description=u"The set of options to pass to MEncoder when transcoding"
+                            )
+
+    ffmpeg2theora_highquality_options = Text(
+                            title = u"ffmpeg2theora command line to produce high-quality OGV files.", 
+                            description=u"The set of options to pass to ffmpeg2theora when transcoding"
+                            )
+
+    nice_level = Text(
+                            title = u"Unix nice level",
+                            description=u"The level to 'renice' the transcoding processes"
+                            )
+
+
 
     file_extensions =  List(
@@ -25 +63 @@
                             )
 
-    flowplayer_URL_SWF = URI (
-                            title=u'URL of Flowplayer SWF',
-                            description=u'Network address of FlowPlayer movie to serve.'
-                            )
-
-    videohost_URL = URI (
-                            title=u'URL of transcoded media server.',
-                            description=u'Network domain name for transcoded media server.'
-                            )
-
-
 class TranscodingQueue(Queue):
     grok.implements(ITranscodingQueue)
 
     #set a sensible default for downloading
-    transcoding_dir=os.path.join(os.getcwd(),'download_queue_files') 
+    transcoding_dir=os.path.join(os.getcwd(),'transcoding_queue_files') 
+
+    mencoder_installed_location = '/usr/local/bin/mencoder'
+    ffmpeg2theora_installed_location = '/usr/bin/ffmpeg2theora'
+    flvtool_installed_location = '/usr/bin/flvtool2'
 
     mencoder_flv_options='-really-quiet -of lavf -oac mp3lame -lameopts abr:br=56 -ovc lavc -lavcopts vcodec=flv:keyint=50:vbitrate=150:mbd=2:mv0:trell:v4mv:cbp:last_pred=3 -vf scale=320:240 -srate 22050'
 
+    mencoder_flv_highquality_options='-really-quiet -of lavf -oac mp3lame -lameopts abr:br=56 -ovc lavc -lavcopts vcodec=flv:keyint=50:vbitrate=250:mbd=2:mv0:trell:v4mv:cbp:last_pred=3 -vf scale=720:480 -srate 22050'
+
     file_extensions=['.avi','.qt','.mov','.mpeg','.mp4','.mpg','.asf','.wmv','.3gp','.m4v','.ogm','.ogg','.divx','flv']
 
-    flowplayer_URL_SWF = 'http://flv.plumi.org/flowplayer/FlowPlayer.swf'
-
-    videohost_URL = ''
+    ffmpeg2theora_options = '-p preview'
+
+    ffmpeg2theora_highquality_options = '-p pro'
+    
+    nice_level = '10'
 
     def __init__(self,id=None):
@@ -63 +98 @@
 
 @grok.subscribe(NetworkMediaItem, grok.IObjectAddedEvent)
+@grok.subscribe(NetworkMediaItem, IJobResetEvent)
 def handle(obj, event):
     #check if the JobItem is STARTING
@@ -78 +114 @@
 
 def attempt_transcode_file(transcoding_queue, networkmediaitem):
-        """Start the transcoding attempt with file 'f'. Converts to FLV format, and OGG/Theora. Produces HTML snippets for including the players appropriate for the produced video format. At the moment, this is FlowPlayer/JW Player for FLV, and Cortado java applet for OGG/Theora"""
+        """Start the transcoding attempt with file 'f'. Converts to FLV format, and OGG/Theora. XXX other mobile formats """
         #whether or not the transcoding worked for the given file 'f'
         worked = False
+
+        #from this queue, we can get our sibling queue 'download'
+        #XXX abstract the names in app.py , and give them constants
+        dld_dir=transcoding_queue.__parent__['download'].download_dir
 
         #get filepath, and then split into stem and extension, and return false straight away if not in recognised set.
@@ -89 +129 @@
 
         #setup filenames we will need.
-        videofile = f
-        lockfile = f+".trans_lock"  # we are encoding already
-        skipfile = f+".trans_skip"  # we tried and failed, don't bother again
-        flvfile  = transcoding_queue.transcoding_dir + stem +  ".flv" # output FLV file
-        theorafile = transcoding_queue.transcoding_dir + ".ogg" # output OGG file
-        includefile  = self.INCLUDE_FILE_DIRECTORY+stem+self.INCLUDE_FILE_SUFFIX
-        logging.info("using %s, %s, %s %s %s " % (lockfile, skipfile, flvfile, theorafile, includefile))
+        original_videofile = f
+        video_file = f.replace(dld_dir,transcoding_queue.transcoding_dir)
+        lockfile = video_file+".trans_lock"  # we are encoding already
+        skipfile = video_file+".trans_skip"  # we tried and failed, don't bother again
+        flvfile  = video_file +  ".flv" # output FLV file
+        theorafile = video_file + ".ogv" # output OGG file
+        logging.info("using %s, %s, %s, %s %s " % (video_file, lockfile, skipfile, flvfile, theorafile))
 
         #check that another encoder isnt already processing this file (lockfile) or that we havent tried and failed before (skipfile)
-        if os.path.exists(skipfile) or os.path(lockfile):
+        if os.path.exists(skipfile) or os.path.exists(lockfile):
             #dont do it
             return False
 
         #OK, valid video file ready to try to transcode
-        logging.info("Checking file %s, using extension %s " % (videofile, extension))
+        logging.info("Using original file %s, using extension %s " % (original_videofile, extension))
         try:
+            #lets make sure all the directories are created
+            try:
+                os.makedirs(os.path.dirname(lockfile))
+            except:
+                #the dirs may already exist
+                pass
+
             # touch the lock file
             #we should catch the exception here, and return FALSE!
@@ -112 +159 @@
                 logging.info("lock file creation failed! ABORTING. %s " % lockfile)
                 return False
-            #lets make sure all the directories are created
-            try:
-                os.makedirs(os.path.dirname(flvfile))
-            except:
-                #the dirs may already exist
-                pass
             
-            logging.info('OK to try encoding: '+videofile)
+            logging.info('OK to try encoding: '+original_videofile)
             #mencoder flv conversion
             start_time=time.time()
-            encoder_command = self.MENCODER_LOCATION + " -quiet " + videofile + " -o " + flvfile + " " + self.MENCODER_OPTIONS
-            encoder_command_string = 'nice -n '+self.BE_HOW_NICE+' '+encoder_command
+            encoder_command = transcoding_queue.mencoder_installed_location + " -quiet " + original_videofile + " -o " + flvfile + " " + transcoding_queue.mencoder_flv_options
+            encoder_command_string = 'nice -n '+ transcoding_queue.nice_level +' '+encoder_command
             output,error = subprocess.Popen([encoder_command_string],shell=True,stdout=subprocess.PIPE,cwd=os.getcwd()).communicate()
             logging.debug("FLV transcoder output: %s ; error: %s" %(output,error))
-            flvtool_command = self.FLVTOOL_LOCATION+" -U stdin "+flvfile
-            flvtool_command_string = "cat "+ flvfile +" | "+ 'nice -n '+ self.BE_HOW_NICE+' '+flvtool_command
+            flvtool_command = transcoding_queue.flvtool_installed_location+" -U stdin "+flvfile
+            flvtool_command_string = "cat "+ flvfile +" | "+ 'nice -n '+ transcoding_queue.nice_level + ' '+flvtool_command
             output,error = subprocess.Popen([flvtool_command_string],shell=True,stdout=subprocess.PIPE,cwd=os.getcwd()).communicate()
             logging.debug("FLV tool output: %s ; error: %s" %(output,error))
             #XXX update the set of transformed file paths on the item.
             finish_time=time.time()
-            logging.info("Encoded %s in %.2f seconds, using cmd -- %s" % (videofile,finish_time-start_time,encoder_command_string))
+            logging.info("Encoded %s in %.2f seconds, using cmd -- %s" % (original_videofile,finish_time-start_time,encoder_command_string))
 
             #ffmpeg2theora , theora/ogg conversion
             start_time=time.time()  
-            theora_cmd =  self.FFMPEG2THEORA_COMMAND + ' ' + videofile + " -o " + theorafile
-            theora_cmd_string='nice -n '+ self.BE_HOW_NICE+' '+ theora_cmd
+            theora_cmd =  transcoding_queue.ffmpeg2theora_installed_location + ' ' + transcoding_queue.ffmpeg2theora_options + " -o " + theorafile + ' ' + original_videofile
+            theora_cmd_string='nice -n '+ transcoding_queue.nice_level+ ' '+ theora_cmd
             output,error = subprocess.Popen([theora_cmd_string],shell=True,stdout=subprocess.PIPE,cwd=os.getcwd()).communicate()
             logging.debug("Theora output: %s ; error: %s" %(output,error))
             #XXX update the set of transformed file paths on the item.
             finish_time=time.time()
-            logging.info("Encoded %s in %.2f seconds, using cmd -- %s" % (videofile,finish_time-start_time,theora_cmd_string))
+            logging.info("Encoded %s in %.2f seconds, using cmd -- %s" % (original_videofile,finish_time-start_time,theora_cmd_string))
 
             if os.path.exists(flvfile) and os.path.getsize(flvfile)>0 and os.path.exists(theorafile) and os.path.getsize(theorafile)>0:
                 # OK ! It Worked.
+
+                #update the object with the details of the new file paths.
+                networkmediaitem.processed_filepaths=(flvfile,theorafile)
+                
                 worked = True
             else:
@@ -154 +199 @@
                 return False
 
-            #Now, make the flash HTML snippet if the flv and Theora got created correctly.
-            #XXX todo, separate out the flv and ogg theora (java applet) html snippet
-            if os.path.exists(flvfile) and os.path.getsize(flvfile)>0:
-                    #lets make sure all the directories are created
-                    try:
-                        os.makedirs(os.path.dirname(includefile))
-                    except:
-                        #the dirs may already exist
-                        pass
-
-                    logging.info("Making html template - original size of %s: %.1fMB, Encoded size: %.1fMB" % (videofile,os.path.getsize(videofile)/1000000.0,os.path.getsize(flvfile)/1000000.0))
-                    data_map={
-                        'flowplayer_location':self.FLOWPLAYER_LOCATION, 
-                        'videofile':flvfile,
-                        'videobaseurl':self.VIDEO_SERVER_URL, 
-                        'splashbaseurl':self.SPLASH_IMAGE_BASE, 
-                        'splashfile':self.SPLASH_IMAGE_FILE, 
-                        'cortado_location':self.CORTADO_LOCATION, 
-                        'oggfile':theorafile,
-                    }
-                    template = Template(file=self.INCLUDE_TEMPLATE, searchList=[data_map])  
-                    file_template=open(includefile, 'w')
-                    file_template.write(template.respond())
-                    file_template.close()
-
-                      
         except:
-            logging.info("Error while processing %s: %s" % (videofile,traceback.format_exc()))
+            logging.info("Error while processing %s: %s" % (original_videofile,traceback.format_exc()))
             #catch exception to remove lock file.
             os.remove(lockfile)
             return False
 
+        os.remove(lockfile)
         #return a status value if we managed to get all the way here.
         return worked