| 34 | | """parse config from the filename argument passed in""" |
| 35 | | config = ConfigParser.RawConfigParser() |
| 36 | | config.read(conf_file) |
| 37 | | self.conf_file = conf_file |
| 38 | | |
| 39 | | |
| 40 | | self.MENCODER_LOCATION=config.get('mencoder','MENCODER_LOCATION') |
| 41 | | self.MENCODER_OPTIONS=config.get('mencoder','MENCODER_OPTIONS') |
| 42 | | |
| 43 | | self.FFMPEG2THEORA_COMMAND=config.get('ffmpeg2theora','FFMPEG2THEORA_COMMAND') |
| 44 | | self.CORTADO_LOCATION=config.get('ffmpeg2theora','CORTADO_LOCATION') |
| 45 | | |
| 46 | | self.FLVTOOL_LOCATION=config.get('flvtool2','FLVTOOL_LOCATION') |
| 47 | | |
| 48 | | self.BE_HOW_NICE=config.get('encoder','BE_HOW_NICE') |
| 49 | | self.CONVERT_THESE=eval(config.get('encoder','CONVERT_THESE')) |
| 50 | | self.DO_ENCODING=config.getboolean('encoder','DO_ENCODING') |
| 51 | | self.NUMBER_OF_PARALLEL_ENCODERS=config.getint('encoder','NUMBER_OF_PARALLEL_ENCODERS') |
| 52 | | self.ENCODER_LOCKFILE_BASE=config.get('encoder','ENCODER_LOCKFILE_BASE') |
| 53 | | self.POLLTIME=int(config.get('encoder','POLLTIME')) |
| 54 | | |
| 55 | | self.VIDEO_FILE_DIRECTORY=config.get('paths','VIDEO_FILE_DIRECTORY') |
| 56 | | self.FLV_FILE_DIRECTORY=config.get('paths','FLV_FILE_DIRECTORY') |
| 57 | | self.INCLUDE_FILE_DIRECTORY=config.get('paths','INCLUDE_FILE_DIRECTORY') |
| 58 | | self.INCLUDE_FILE_SUFFIX=config.get('paths','INCLUDE_FILE_SUFFIX') |
| 59 | | self.INCLUDE_TEMPLATE=config.get('paths','INCLUDE_TEMPLATE') |
| 60 | | |
| 61 | | self.FLOWPLAYER_LOCATION=config.get('urls','FLOWPLAYER_LOCATION') |
| 62 | | self.VIDEO_SERVER_URL=config.get('urls','VIDEO_SERVER_URL') |
| 63 | | self.SPLASH_IMAGE_BASE=config.get('urls','SPLASH_IMAGE_BASE') |
| 64 | | self.SPLASH_IMAGE_FILE=config.get('urls','SPLASH_IMAGE_FILE') |
| 65 | | |
| 66 | | self.LOG_FILE=config.get('logging','LOG_FILE') |
| 67 | | self.LOG_LEVEL=eval(config.get('logging','LOG_LEVEL')) |
| 68 | | |
| 69 | | self.ENCODER_LOCKFILE = "" |
| 70 | | |
| 71 | | logging.basicConfig(level=self.LOG_LEVEL, format='%(asctime)s %(levelname)s %(message)s', filename=self.LOG_FILE, filemode='a') |
| 72 | | |
| 73 | | logging.info("finished parse_config function at %s, using file %s " % (time.strftime("%D %H:%M:%S"), self.conf_file)) |
| | 40 | """parse config from the filename argument passed in""" |
| | 41 | config = ConfigParser.RawConfigParser() |
| | 42 | config.read(conf_file) |
| | 43 | self.conf_file = conf_file |
| | 44 | |
| | 45 | |
| | 46 | self.MENCODER_LOCATION=config.get('mencoder','MENCODER_LOCATION') |
| | 47 | self.MENCODER_OPTIONS=config.get('mencoder','MENCODER_OPTIONS') |
| | 48 | |
| | 49 | self.FFMPEG2THEORA_COMMAND=config.get('ffmpeg2theora','FFMPEG2THEORA_COMMAND') |
| | 50 | self.CORTADO_LOCATION=config.get('ffmpeg2theora','CORTADO_LOCATION') |
| | 51 | |
| | 52 | self.FLVTOOL_LOCATION=config.get('flvtool2','FLVTOOL_LOCATION') |
| | 53 | |
| | 54 | self.BE_HOW_NICE=config.get('encoder','BE_HOW_NICE') |
| | 55 | self.CONVERT_THESE=eval(config.get('encoder','CONVERT_THESE')) |
| | 56 | self.DO_ENCODING=config.getboolean('encoder','DO_ENCODING') |
| | 57 | self.NUMBER_OF_PARALLEL_ENCODERS=config.getint('encoder','NUMBER_OF_PARALLEL_ENCODERS') |
| | 58 | self.ENCODER_LOCKFILE_BASE=config.get('encoder','ENCODER_LOCKFILE_BASE') |
| | 59 | self.POLLTIME=int(config.get('encoder','POLLTIME')) |
| | 60 | |
| | 61 | self.VIDEO_FILE_DIRECTORY=config.get('paths','VIDEO_FILE_DIRECTORY') |
| | 62 | self.FLV_FILE_DIRECTORY=config.get('paths','FLV_FILE_DIRECTORY') |
| | 63 | self.INCLUDE_FILE_DIRECTORY=config.get('paths','INCLUDE_FILE_DIRECTORY') |
| | 64 | self.TORRENT_DIRECTORY=config.get('paths','TORRENT_DIRECTORY') |
| | 65 | |
| | 66 | self.INCLUDE_FILE_SUFFIX=config.get('paths','INCLUDE_FILE_SUFFIX') |
| | 67 | self.INCLUDE_TEMPLATE=config.get('paths','INCLUDE_TEMPLATE') |
| | 68 | |
| | 69 | |
| | 70 | self.FLOWPLAYER_LOCATION=config.get('urls','FLOWPLAYER_LOCATION') |
| | 71 | self.VIDEO_SERVER_URL=config.get('urls','VIDEO_SERVER_URL') |
| | 72 | self.SPLASH_IMAGE_BASE=config.get('urls','SPLASH_IMAGE_BASE') |
| | 73 | self.SPLASH_IMAGE_FILE=config.get('urls','SPLASH_IMAGE_FILE') |
| | 74 | |
| | 75 | self.LOG_FILE=config.get('logging','LOG_FILE') |
| | 76 | #whoa, eval , from a conf file input. nasty |
| | 77 | self.LOG_LEVEL=eval(config.get('logging','LOG_LEVEL')) |
| | 78 | |
| | 79 | self.DOWNLOAD_QUEUE=config.get('zopetube-server','DOWNLOAD_QUEUE') |
| | 80 | self.BITTORRENT_QUEUE=config.get('zopetube-server','BITTORRENT_QUEUE') |
| | 81 | self.TRANSCODING_QUEUE=config.get('zopetube-server','TRANSCODING_QUEUE') |
| | 82 | |
| | 83 | #bittorrent |
| | 84 | self.TRACKER_URL=config.get('bittorrent','TRACKER_URL') |
| | 85 | |
| | 86 | self.ENCODER_LOCKFILE = "" |
| | 87 | |
| | 88 | logging.basicConfig(level=self.LOG_LEVEL, format='%(asctime)s %(levelname)s %(message)s', filename=self.LOG_FILE, filemode='a') |
| | 89 | |
| | 90 | logging.info("finished parse_config function at %s, using file %s " % (time.strftime("%D %H:%M:%S"), self.conf_file)) |
| 76 | | """ inits the logging, and checks for the parallel lock files. If there are no more free spots as an encoder, returns False, else returns True""" |
| 77 | | self.ENCODER_LOCKFILE="" #this is dynamic, generated off the base |
| 78 | | |
| 79 | | #check we arent already running (up to the number of parallel encoders) and if we are , exit |
| 80 | | for n in range(0,self.NUMBER_OF_PARALLEL_ENCODERS): |
| 81 | | #set the lockfile name, for later |
| 82 | | self.ENCODER_LOCKFILE="%s.%s" % (self.ENCODER_LOCKFILE_BASE,n) |
| 83 | | if os.path.exists(self.ENCODER_LOCKFILE): |
| 84 | | if n==(self.NUMBER_OF_PARALLEL_ENCODERS-1): |
| 85 | | logging.info("Max encoders reached(%s), exiting." % self.NUMBER_OF_PARALLEL_ENCODERS) |
| 86 | | #we should exit the program here |
| 87 | | #sys.exit("Max encoders reached(%s), exiting." % self.NUMBER_OF_PARALLEL_ENCODERS) |
| 88 | | return False |
| 89 | | else: |
| 90 | | #we have a free spot , as encoder 'n', lets make the lock file and break out |
| 91 | | os.mknod(self.ENCODER_LOCKFILE) |
| 92 | | break |
| 93 | | return True |
| | 93 | """ inits the logging, and checks for the parallel lock files. If there are no more free spots as an encoder, returns False, else returns True""" |
| | 94 | self.ENCODER_LOCKFILE="" #this is dynamic, generated off the base |
| | 95 | |
| | 96 | #check we arent already running (up to the number of parallel encoders) and if we are , exit |
| | 97 | for n in range(0,self.NUMBER_OF_PARALLEL_ENCODERS): |
| | 98 | #set the lockfile name, for later |
| | 99 | self.ENCODER_LOCKFILE="%s.%s" % (self.ENCODER_LOCKFILE_BASE,n) |
| | 100 | if os.path.exists(self.ENCODER_LOCKFILE): |
| | 101 | if n==(self.NUMBER_OF_PARALLEL_ENCODERS-1): |
| | 102 | logging.info("Max encoders reached(%s), exiting." % self.NUMBER_OF_PARALLEL_ENCODERS) |
| | 103 | #we should exit the program here |
| | 104 | #sys.exit("Max encoders reached(%s), exiting." % self.NUMBER_OF_PARALLEL_ENCODERS) |
| | 105 | return False |
| | 106 | else: |
| | 107 | #we have a free spot , as encoder 'n', lets make the lock file and break out |
| | 108 | os.mknod(self.ENCODER_LOCKFILE) |
| | 109 | break |
| | 110 | return True |
| | 111 | |
| | 112 | def do_torrent_loop(self): |
| | 113 | """do one torrent loop""" |
| | 114 | logging.info("Starting indytube torrent component... in %s " % self.TORRENT_DIRECTORY) |
| | 115 | self.available = 0 |
| | 116 | self.downloaded = 0 |
| | 117 | |
| | 118 | #download them from the server. |
| | 119 | try: |
| | 120 | self.xmlrpc_ref = xmlrpclib.Server(self.BITTORRENT_QUEUE) |
| | 121 | self.jobs = self.xmlrpc_ref.listofWaitingJobItems() |
| | 122 | except: |
| | 123 | logging.info("Ending indytube downloading component because of exception!") |
| | 124 | return |
| | 125 | #processing.... |
| | 126 | self.available = len(self.jobs) |
| | 127 | logging.info("We have %s files to make torrent files for, and then seed." % self.available) |
| | 128 | for v in self.jobs: |
| | 129 | ###XXX put this block of work per job into a callback, and fire async. |
| | 130 | ok,file = self.torrent_and_seed_file(v) |
| | 131 | if ok: |
| | 132 | self.downloaded = self.downloaded + 1 |
| | 133 | self.xmlrpc_ref.markAsFinished(v['id']) |
| | 134 | else: |
| | 135 | self.xmlrpc_ref.markAsFailed(v['id']) |
| | 136 | |
| | 137 | |
| | 138 | logging.info("Ending indytube torrent component... We got a list of %s eligble files, torrent-ed %s files " % (self.available, self.downloaded)) |
| | 139 | |
| | 140 | ## end : do_torrent_loop |
| | 141 | |
| | 142 | def torrent_and_seed_file(self,jobitem_dict): |
| | 143 | rel_fp = jobitem_dict['video_uri'] |
| | 144 | incoming_filename = self.VIDEO_FILE_DIRECTORY + rel_fp |
| | 145 | target_torrent = self.TORRENT_DIRECTORY + rel_fp + ".torrent" |
| | 146 | |
| | 147 | logging.info("torrent and seed: media file from location %s into location %s " % (incoming_filename,target_torrent)) |
| | 148 | #lets make sure all the directories are created |
| | 149 | try: |
| | 150 | os.makedirs(os.path.dirname(target_torrent)) |
| | 151 | except: |
| | 152 | #the dirs may already exist |
| | 153 | pass |
| | 154 | output = "" |
| | 155 | error = "" |
| | 156 | try: |
| | 157 | video_file = incoming_filename |
| | 158 | tracker_url = self.TRACKER_URL |
| | 159 | #maketorrent-console API |
| | 160 | #maketorrent-console --target .torrent-file tracker_url video_file |
| | 161 | make_torrent_args = "maketorrent-console --target %s %s %s " % (target_torrent, tracker_url, video_file) |
| | 162 | logging.debug("maketorrent-console input: %s ; cwd : %s" % (make_torrent_args,os.getcwd())) |
| | 163 | output,error = subprocess.Popen([make_torrent_args],shell=True,stdout=subprocess.PIPE, cwd=os.getcwd()).communicate() |
| | 164 | |
| | 165 | logging.debug("maketorrent-console output and error : %s %s" % (output,error)) |
| | 166 | |
| | 167 | #bittorrent-console API |
| | 168 | #bittorrent-console --save_as video_file torrent_file |
| | 169 | bittorrent_args = "bittorrent-console --save_as %s %s" % (video_file,target_torrent) |
| | 170 | logging.debug("bittorrent-console input: %s ; cwd : %s" % (bittorrent_args,os.getcwd())) |
| | 171 | subprocess.Popen([bittorrent_args],shell=True,cwd=os.getcwd()) |
| | 172 | |
| | 173 | logging.debug("bittorrent-console output and error : %s %s" % (output,error)) |
| | 174 | |
| | 175 | |
| | 176 | except Exception,e: |
| | 177 | logging.error(e) |
| | 178 | logging.debug("maketorrent-console output and error : %s %s" % (output,error)) |
| | 179 | return [False,None] |
| | 180 | #XXX make md5sum and compare it |
| | 181 | return [True,rel_fp] |
| | 182 | |
| | 183 | |
| | 184 | |
| | 185 | def do_downloading_loop(self): |
| | 186 | """do one downloading loop""" |
| | 187 | logging.info("Starting indytube downloading component... in %s " % self.VIDEO_FILE_DIRECTORY) |
| | 188 | self.available = 0 |
| | 189 | self.downloaded = 0 |
| | 190 | |
| | 191 | #download them from the server. |
| | 192 | try: |
| | 193 | self.xmlrpc_ref = xmlrpclib.Server(self.DOWNLOAD_QUEUE) |
| | 194 | self.jobs = self.xmlrpc_ref.listofWaitingJobItems() |
| | 195 | except: |
| | 196 | logging.info("Ending indytube downloading component because of exception!") |
| | 197 | return |
| | 198 | #processing.... |
| | 199 | self.available = len(self.jobs) |
| | 200 | logging.info("We have %s files to download." % self.available) |
| | 201 | for v in self.jobs: |
| | 202 | ###XXX put this block of work per job into a callback, and fire async. |
| | 203 | ok,file = self.download_file(v) |
| | 204 | if ok: |
| | 205 | self.downloaded = self.downloaded + 1 |
| | 206 | self.xmlrpc_ref.markAsFinished(v['id']) |
| | 207 | #add in new job item on transcoding queue |
| | 208 | self.xmlrpc_ref_t = xmlrpclib.Server(self.TRANSCODING_QUEUE) |
| | 209 | self.xmlrpc_ref_t.addJobItem(file,'') |
| | 210 | else: |
| | 211 | self.xmlrpc_ref.markAsFailed(v['id']) |
| | 212 | |
| | 213 | |
| | 214 | logging.info("Ending indytube downloading component... We got a list of %s eligble files, downloaded %s files " % (self.available, self.downloaded)) |
| | 215 | |
| | 216 | ## end : do_downloading_loop |
| | 217 | |
| | 218 | def download_file(self,jobitem_dict): |
| | 219 | download_url = jobitem_dict['video_uri'] |
| | 220 | url_parts = urlparse(download_url) |
| | 221 | rel_fp = url_parts[1]+url_parts[2] |
| | 222 | incoming_filename = self.VIDEO_FILE_DIRECTORY + rel_fp |
| | 223 | logging.debug("url parts are %s %s %s %s %s %s" % (url_parts[0],url_parts[1],url_parts[2],url_parts[3],url_parts[4],url_parts[5])) |
| | 224 | logging.info("downloading media file from url %s, into location %s " % (download_url,incoming_filename)) |
| | 225 | #lets make sure all the directories are created |
| | 226 | try: |
| | 227 | os.makedirs(os.path.dirname(incoming_filename)) |
| | 228 | except: |
| | 229 | #the dirs may already exist |
| | 230 | pass |
| | 231 | try: |
| | 232 | filename,headers = urllib.urlretrieve(download_url, incoming_filename) |
| | 233 | except: |
| | 234 | logging.info("downloading media file from url %s, into location %s in error! " % (download_url,incoming_filename)) |
| | 235 | return [False,None] |
| | 236 | #XXX make md5sum and compare it |
| | 237 | return [True,rel_fp] |
| | 238 | |
| 96 | | """do one transcoding loop""" |
| 97 | | logging.info("Starting indytube... in %s " % self.VIDEO_FILE_DIRECTORY) |
| 98 | | self.checked = 0 |
| 99 | | self.converted = 0 |
| 100 | | for root,dir,files in os.walk(self.VIDEO_FILE_DIRECTORY): |
| 101 | | for f in files: |
| 102 | | ok = self.attempt_transcode_file(f) |
| 103 | | if ok: |
| 104 | | self.converted = self.converted + 1 |
| 105 | | |
| 106 | | #remove this process's lockfile, we have finished the loop |
| 107 | | os.remove(self.ENCODER_LOCKFILE) |
| 108 | | logging.info("Ending indytube... We checked %s eligble files, converted %s files " % (self.checked, self.converted)) |
| 109 | | |
| 110 | | ## end : do_transcoding_loop |
| 111 | | |
| 112 | | def attempt_transcode_file(self, f): |
| 113 | | '''Start the transcoding attempt with file 'f'. Converts to FLV format, and OGG/Theora. Produces HTML snippets for including |
| 114 | | the players appropriate for the produced video format. At the moment, this is FlowPlayer for FLV, and Cortado java applet for OGG/Theora ''' |
| 115 | | #whether or not the transcoding worked for the given file 'f' |
| 116 | | worked = False |
| 117 | | |
| 118 | | (stem,extension)=os.path.splitext(f) |
| 119 | | if extension.lower() in self.CONVERT_THESE: #we should convert the file |
| 120 | | |
| 121 | | #get last character, or first one |
| 122 | | relative_directory=self.VIDEO_FILE_DIRECTORY[len(self.VIDEO_FILE_DIRECTORY):] |
| 123 | | if relative_directory.startswith(os.sep): |
| 124 | | relative_directory=relative_directory[1:] # make sure we are relative |
| 125 | | |
| 126 | | root = self.VIDEO_FILE_DIRECTORY |
| 127 | | videofile = os.path.join(root,f) |
| 128 | | lockfile = os.path.join(root,stem+".wetube_lock") # we are encoding already |
| 129 | | skipfile = os.path.join(root,stem+".wetube_skip") # we tried and failed, don't bother again |
| 130 | | flvfile = os.path.join(self.FLV_FILE_DIRECTORY,relative_directory,stem+".flv") |
| 131 | | theorafile = os.path.join(self.FLV_FILE_DIRECTORY,relative_directory,stem+".ogg") |
| 132 | | includefile = os.path.join(self.INCLUDE_FILE_DIRECTORY,relative_directory,stem+self.INCLUDE_FILE_SUFFIX) |
| 133 | | logging.info("check for %s, %s, %s " % (lockfile, skipfile, flvfile)) |
| 134 | | |
| 135 | | #check that another encoder isnt already processing this file (lockfile) or that we havent tried and failed before (skipfile) |
| 136 | | if not(os.path.exists(lockfile) or os.path.exists(skipfile)): |
| 137 | | #OK, valid video file ready to try to transcode |
| 138 | | logging.info("Checking file %s, using extension %s " % ( os.path.join(root,f), extension)) |
| 139 | | self.checked = self.checked + 1 |
| 140 | | try: |
| 141 | | # touch the lock file |
| 142 | | os.mknod(lockfile) |
| 143 | | |
| 144 | | # if the flv file (autogenerated) or html snippet is not there, then reencode! |
| 145 | | if not(os.path.exists(flvfile)) or not(os.path.exists(includefile)): |
| 146 | | logging.info('OK to try encoding: '+videofile) |
| 147 | | |
| 148 | | #pipe_to_null = '> /dev/null 2>&1' |
| 149 | | if self.DO_ENCODING: #maybe we just want to regenerate the include file! |
| 150 | | #mencoder flv conversion |
| 151 | | start_time=time.time() |
| 152 | | encoder_command = self.MENCODER_LOCATION + " -quiet " + videofile + " -o " + flvfile + " " + self.MENCODER_OPTIONS |
| 153 | | os.system('nice -n '+self.BE_HOW_NICE+' '+encoder_command) |
| 154 | | finish_time=time.time() |
| 155 | | logging.info("Encoded %s in %.2f seconds, using cmd -- %s" % (videofile,finish_time-start_time,encoder_command)) |
| 156 | | flvtool_command = self.FLVTOOL_LOCATION+" -U stdin "+flvfile |
| 157 | | os.system("cat "+ flvfile +" | "+ 'nice -n '+ self.BE_HOW_NICE+' '+flvtool_command) |
| 158 | | |
| 159 | | #ffmpeg2theora , theora/ogg conversion |
| 160 | | start_time=time.time() |
| 161 | | theora_cmd = self.FFMPEG2THEORA_COMMAND + ' ' + videofile + " -o " + theorafile |
| 162 | | os.system('nice -n '+ self.BE_HOW_NICE+' '+ theora_cmd) |
| 163 | | finish_time=time.time() |
| 164 | | logging.info("Encoded %s in %.2f seconds, using cmd -- %s" % (videofile,finish_time-start_time,theora_cmd)) |
| 165 | | |
| 166 | | if os.path.exists(flvfile) and os.path.getsize(flvfile)>0: |
| 167 | | # OK ! It Worked. |
| 168 | | #XXX expand to OGG format checking |
| 169 | | worked = True |
| 170 | | |
| 171 | | else: |
| 172 | | logging.debug("skipped encoding, will just do html template generation, if flv exists as non-zero size") |
| | 241 | """do one transcoding loop""" |
| | 242 | logging.info("Starting indytube transcoding loop ... ") |
| | 243 | self.checked = 0 |
| | 244 | self.converted = 0 |
| | 245 | |
| | 246 | #download the batch of job items from the server. |
| | 247 | try: |
| | 248 | self.xmlrpc_ref = xmlrpclib.Server(self.TRANSCODING_QUEUE) |
| | 249 | self.jobs = self.xmlrpc_ref.listofWaitingJobItems() |
| | 250 | except: |
| | 251 | logging.info("Ending indytube transcoding component because of exception") |
| | 252 | return |
| | 253 | |
| | 254 | #processing.... |
| | 255 | logging.info("We have %s files to transcode." % len(self.jobs)) |
| | 256 | for v in self.jobs: |
| | 257 | ###XXX put this block of work per job, into a callback, and fire async. |
| | 258 | #assumes video_uri is the relative filepath from the downloading server. |
| | 259 | ok, file = self.attempt_transcode_file(v) |
| | 260 | if ok: |
| | 261 | self.converted = self.converted + 1 |
| | 262 | self.xmlrpc_ref.markAsFinished(v['id']) |
| | 263 | #lets mark it as ready for torrent-ing and seeding |
| | 264 | self.xmlrpc_ref_bt = xmlrpclib.Server(self.BITTORRENT_QUEUE) |
| | 265 | self.xmlrpc_ref_bt.addJobItem(file,'') |
| | 266 | |
| | 267 | else: |
| | 268 | self.xmlrpc_ref.markAsFailed(v['id']) |
| | 269 | |
| | 270 | logging.info("Ending indytube transcoding loop ... We checked %s eligble files, converted %s files " % (self.checked, self.converted)) |
| | 271 | |
| | 272 | ## end : do_transcoding_loop |
| | 273 | |
| | 274 | def attempt_transcode_file(self, jobitem_dict): |
| | 275 | """Start the transcoding attempt with file 'f'. Converts to FLV format, and OGG/Theora. Produces HTML snippets for including the players appropriate for the produced video format. At the moment, this is FlowPlayer for FLV, and Cortado java applet for OGG/Theora""" |
| | 276 | #whether or not the transcoding worked for the given file 'f' |
| | 277 | worked = False |
| | 278 | |
| | 279 | f = jobitem_dict['video_uri'] |
| | 280 | |
| | 281 | (stem,extension)=os.path.splitext(f) |
| | 282 | if extension.lower() in self.CONVERT_THESE: #we should convert the file |
| | 283 | |
| | 284 | videofile = self.VIDEO_FILE_DIRECTORY + f |
| | 285 | lockfile = videofile+".wetube_lock" # we are encoding already |
| | 286 | skipfile = videofile+".wetube_skip" # we tried and failed, don't bother again |
| | 287 | flvfile = self.FLV_FILE_DIRECTORY+stem+".flv" |
| | 288 | theorafile = self.FLV_FILE_DIRECTORY+stem+".ogg" |
| | 289 | includefile = self.INCLUDE_FILE_DIRECTORY+stem+self.INCLUDE_FILE_SUFFIX |
| | 290 | logging.info("check for %s, %s, %s " % (lockfile, skipfile, flvfile)) |
| | 291 | |
| | 292 | #check that another encoder isnt already processing this file (lockfile) or that we havent tried and failed before (skipfile) |
| | 293 | if os.path.exists(skipfile): |
| | 294 | #lets let the transcoding server know we are in error |
| | 295 | self.xmlrpc_ref.markAsFailed(jobitem_dict['id']) |
| | 296 | |
| | 297 | if not(os.path.exists(lockfile) or os.path.exists(skipfile)): |
| | 298 | #OK, valid video file ready to try to transcode |
| | 299 | logging.info("Checking file %s, using extension %s " % ( videofile, extension)) |
| | 300 | self.checked = self.checked + 1 |
| | 301 | try: |
| | 302 | # touch the lock file |
| | 303 | os.mknod(lockfile) |
| | 304 | |
| | 305 | # if the flv file (autogenerated) or html snippet is not there, then reencode! |
| | 306 | if not(os.path.exists(flvfile)) or not(os.path.exists(includefile)): |
| | 307 | logging.info('OK to try encoding: '+videofile) |
| | 308 | #lets make sure all the directories are created |
| | 309 | try: |
| | 310 | os.makedirs(os.path.dirname(flvfile)) |
| | 311 | except: |
| | 312 | #the dirs may already exist |
| | 313 | pass |
| | 314 | |
| | 315 | #pipe_to_null = '> /dev/null 2>&1' |
| | 316 | if self.DO_ENCODING: #maybe we just want to regenerate the include file! |
| | 317 | #mencoder flv conversion |
| | 318 | start_time=time.time() |
| | 319 | encoder_command = self.MENCODER_LOCATION + " -quiet " + videofile + " -o " + flvfile + " " + self.MENCODER_OPTIONS |
| | 320 | encoder_command_string = 'nice -n '+self.BE_HOW_NICE+' '+encoder_command |
| | 321 | output,error = subprocess.Popen([encoder_command_string],shell=True,stdout=subprocess.PIPE,cwd=os.getcwd()).communicate() |
| | 322 | logging.debug("FLV transcoder output: %s ; error: %s" %(output,error)) |
| | 323 | flvtool_command = self.FLVTOOL_LOCATION+" -U stdin "+flvfile |
| | 324 | flvtool_command_string = "cat "+ flvfile +" | "+ 'nice -n '+ self.BE_HOW_NICE+' '+flvtool_command |
| | 325 | output,error = subprocess.Popen([flvtool_command_string],shell=True,stdout=subprocess.PIPE,cwd=os.getcwd()).communicate() |
| | 326 | logging.debug("FLV tool output: %s ; error: %s" %(output,error)) |
| | 327 | |
| | 328 | finish_time=time.time() |
| | 329 | logging.info("Encoded %s in %.2f seconds, using cmd -- %s" % (videofile,finish_time-start_time,encoder_command_string)) |
| | 330 | #ffmpeg2theora , theora/ogg conversion |
| | 331 | start_time=time.time() |
| | 332 | theora_cmd = self.FFMPEG2THEORA_COMMAND + ' ' + videofile + " -o " + theorafile |
| | 333 | theora_cmd_string='nice -n '+ self.BE_HOW_NICE+' '+ theora_cmd |
| | 334 | output,error = subprocess.Popen([theora_cmd_string],shell=True,stdout=subprocess.PIPE,cwd=os.getcwd()).communicate() |
| | 335 | logging.debug("Theora output: %s ; error: %s" %(output,error)) |
| | 336 | |
| | 337 | finish_time=time.time() |
| | 338 | logging.info("Encoded %s in %.2f seconds, using cmd -- %s" % (videofile,finish_time-start_time,theora_cmd_string)) |
| | 339 | |
| | 340 | if os.path.exists(flvfile) and os.path.getsize(flvfile)>0: |
| | 341 | # OK ! It Worked. |
| | 342 | #XXX expand to OGG format checking |
| | 343 | worked = True |
| | 344 | |
| | 345 | else: |
| | 346 | logging.debug("skipped encoding, will just do html template generation, if flv exists as non-zero size") |
| 174 | | else: |
| 175 | | logging.debug("flv file and html file already exists, not doing transcoding") |
| 176 | | |
| 177 | | #make the flash HTML snippet if the flv got created correctly. |
| 178 | | #XXX todo, separate out the flv and ogg theora (java applet) html snippet |
| 179 | | if os.path.exists(flvfile) and os.path.getsize(flvfile)>0: |
| 180 | | logging.info("Making html template - original size of %s: %.1fMB, Encoded size: %.1fMB" % (videofile,os.path.getsize(videofile)/1000000.0,os.path.getsize(flvfile)/1000000.0)) |
| 181 | | data_map={ |
| 182 | | 'flowplayer_location':self.FLOWPLAYER_LOCATION, |
| 183 | | 'videofile':relative_directory+'/'+stem+".flv", |
| 184 | | 'videobaseurl':self.VIDEO_SERVER_URL, |
| 185 | | 'splashbaseurl':self.SPLASH_IMAGE_BASE, |
| 186 | | 'splashfile':self.SPLASH_IMAGE_FILE, |
| 187 | | 'cortado_location':self.CORTADO_LOCATION, |
| 188 | | 'oggfile':stem+".ogg", |
| 189 | | 'mirid':stem |
| 190 | | } |
| 191 | | t = Template(file=self.INCLUDE_TEMPLATE, searchList=[data_map]) |
| 192 | | f=open(includefile, 'w') |
| 193 | | f.write(t.respond()) |
| 194 | | f.close() |
| 195 | | |
| 196 | | else: |
| 197 | | logging.info("FLV file size is zero - assuming encoding failed! Permanently skipping file!") |
| 198 | | os.mknod(skipfile) |
| | 348 | else: |
| | 349 | logging.debug("flv file and html file already exists, not doing transcoding") |
| | 350 | |
| | 351 | #Now, make the flash HTML snippet if the flv got created correctly. |
| | 352 | #XXX todo, separate out the flv and ogg theora (java applet) html snippet |
| | 353 | if os.path.exists(flvfile) and os.path.getsize(flvfile)>0: |
| | 354 | #lets make sure all the directories are created |
| | 355 | try: |
| | 356 | os.makedirs(os.path.dirname(includefile)) |
| | 357 | except: |
| | 358 | #the dirs may already exist |
| | 359 | pass |
| | 360 | |
| | 361 | logging.info("Making html template - original size of %s: %.1fMB, Encoded size: %.1fMB" % (videofile,os.path.getsize(videofile)/1000000.0,os.path.getsize(flvfile)/1000000.0)) |
| | 362 | data_map={ |
| | 363 | 'flowplayer_location':self.FLOWPLAYER_LOCATION, |
| | 364 | 'videofile':flvfile, |
| | 365 | 'videobaseurl':self.VIDEO_SERVER_URL, |
| | 366 | 'splashbaseurl':self.SPLASH_IMAGE_BASE, |
| | 367 | 'splashfile':self.SPLASH_IMAGE_FILE, |
| | 368 | 'cortado_location':self.CORTADO_LOCATION, |
| | 369 | 'oggfile':theorafile, |
| | 370 | 'mirid':stem |
| | 371 | } |
| | 372 | template = Template(file=self.INCLUDE_TEMPLATE, searchList=[data_map]) |
| | 373 | file_template=open(includefile, 'w') |
| | 374 | file_template.write(template.respond()) |
| |
