From: Alan J. Pippin <alan@pippins.net>
Date: Sat, 24 Dec 2011 22:37:31 +0000 (-0700)
Subject: Major changes to support interlaced/progressive content
X-Git-Url: http://git.pippins.net/embedvideo/.git/static/images/%7Bdelete%7D?a=commitdiff_plain;h=d0bb8d7ba0914fb2e31422aa1207ba6eaf490deb;p=videoscripts%2F.git

Major changes to support interlaced/progressive content
Major changes to support re-quantization to significantly downsize large HD video streams with little impact to quality.
---

diff --git a/make_mkv b/make_mkv
index 04556e5..c4f3707 100755
--- a/make_mkv
+++ b/make_mkv
@@ -18,22 +18,23 @@ use DateTime::Format::Duration;
 
 ####################################################################################################
 # Configuration parameters - CHANGE THESE TO SUITE YOUR NEEDS
-my $mkvmerge=`which mkvmerge`; chomp($mkvmerge);
-my $ffmpeg=`which ffmpeg`; chomp($ffmpeg);
+my $mkvmerge='mkvmerge';
+my $ffmpeg='ffmpeg';
+my $handbrake='HandBrakeCLI';
 my $tmpfile = `tempfile`; chomp($tmpfile);
 my $chapter_file = $tmpfile;
 my $input_file_options = "-S";
 my $output_file_options = "--chapters $chapter_file --compression -1:none";
 my $timezone = `cat /etc/timezone`; chomp($timezone);
+my $interlaced_requantize_quality=0.85;
+my $progressive_requantize_quality=0.7;
 ####################################################################################################
 
 
 ####################################################################################################
 # Command Line Options
-getopts("st:o:i:h");
+getopts("sqt:o:i:h");
 
-if(! -x $mkvmerge) { die "-E- Unable to find required program: mkvmerge\n"; }
-if(! -x $ffmpeg) { die "-E- Unable to find required program: ffmpeg\n"; }
 if(! defined $opt_t) { &usage(); die "-E- Missing required title: -t <title>\n"; }
 if(! defined $opt_o) { &usage(); die "-E- Missing required argument output video names: -o <output.mkv>\n"; }
 if(! defined $opt_i) { &usage(); die "-E- Missing required argument input video names: -i <input,input,...>\n"; }
@@ -43,6 +44,7 @@ sub usage {
     print "  -t <title>            Sets the general title for the output video file\n";
     print "  -o <output.mkv>       Sets the name of the output mkv file\n";
     print "  -i <input,input,...>  Sets the name of the input files to make into an mkv file\n";
+    print "  -q                    Requantize MTS input videos to decrease output video size (requires HandBrakeCLI)\n";
     print "  -s                    Simulate mode. Don't actually make the video, but tell us what you will do\n";
     print "\n";
     return 1;
@@ -111,22 +113,24 @@ my $cmd = "$mkvmerge --title \"$opt_t\" $output_file_options -o \"$opt_o\"";
 my $video_count = 0;
 my @video_tmp_files;
 foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) {
+    # Make a note of the video extension
+    my $video_ext = $video;
+    $video_ext =~ s/.*\.(\S+)$/$1/;
+
     # Detect if the input file is interlaced or not.
     # There is a bug in mkvmerge 5.0.1 and earlier where if the video content is 1080i, it doesn't mux it properly, and it stutters.
     # The quick solution to this issue is to run the interlaced video file through ffmpeg and tell it to copy the video/audio streams to a mkv container.
     # We will then merge this temporarily created mkv container into the final mkv container instead of the original interlaced video.
     # http://ubuntuforums.org/showthread.php?t=1627194
     # http://forum.doom9.org/showthread.php?t=155732&page=31
-    my $interlaced = system('ffmpeg -i "$video" 2>&1 | grep -q "frame rate differs"');
-    if($interlaced != 0) {
+    my $progressive = system('$ffmpeg -i "$video" 2>&1 | grep -q "frame rate differs"');
+    if(!$progressive) {
 	my $video_mkv = $video;
-	my $video_ext = $video;
-	$video_ext =~ s/.*\.(\S+)$/$1/;
 	print "   Detected interlaced video content: $video\n";
 	if($video_ext !~ /mkv/i) {
-	    $video_mkv =~ s/\.[^.]*$//; $video_mkv .= ".mkv";
+	    $video_mkv =~ s/\.[^.]*$//; $video_mkv .= ".ffmpeg.mkv";
 	    print "   Re-muxing the interlaced video content as an mkv file: $video_mkv\n";
-	    my $make_mkv_cmd = "ffmpeg -i \"$video\" -scodec copy -acodec copy -vcodec copy -f matroska \"$video_mkv\" > /dev/null 2>&1";
+	    my $make_mkv_cmd = "$ffmpeg -y -i \"$video\" -scodec copy -acodec copy -vcodec copy -f matroska \"$video_mkv\" > /dev/null 2>&1";
 	    if(! defined $opt_s) { 
 		my $errno = system("$make_mkv_cmd");
 		$errno = $errno >> 8;
@@ -143,7 +147,41 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) {
     } else { 
 	print "   Detected progressive video content: $video\n";
     }
-    # Create our intput file command line options for this video
+
+    # Re-quantize the input video to reduce the resulting output filesize
+    # This also gives us a chance to deinterlace the video as well
+    # Only do this for .MTS videos
+    if((defined $opt_q) && ($video_ext =~ /mts/i)) {
+
+	# Set our requantize factor accordingly
+	my $requantize_option = "";
+	if(!$progressive) { $requantize_option = "-q $interlaced_requantize_quality"; }
+	else { $requantize_option = "-q $progressive_requantize_quality"; }
+	
+	# Set our de-interlace option accordingly
+	my $deinterlace_option = "";
+	if(!$progressive) { $deinterlace_option = "-d"; }
+
+	# Use HandBrake to requantize/deinterlace the input video
+	my $video_mp4 = $video; $video_mp4 =~ s/\.[^.]*$//; $video_mp4 .= ".hb.mp4";
+	print "   Re-quantizing input video content: $video_mp4\n";
+	my $handbrake_cmd = "$handbrake $deinterlace_option $requantize_option -E ac3 -i \"$video\" -o \"$video_mp4\" > /dev/null 2>&1";
+	if(! defined $opt_s) { 
+	    my $errno = system("$handbrake_cmd");
+	    $errno = $errno >> 8;
+	    if($errno > 1) {
+		unlink "$video_mp4";
+		die "-E- handbrake encountered some errors with exit code $errno\n";
+	    }
+	}
+	
+	# Push the name of our intermediate video file onto a list of files to delete on exit
+	push(@video_tmp_files, "$video_mp4");
+	# Update the name of our video file to equal the name of our new intermediate video file name
+	$video = $video_mp4;
+    }
+    
+    # Create our input file command line options for this video
     if($video_count == 0) {
 	$cmd .= " $input_file_options \"$video\"";
     } else {
diff --git a/merge_videos_by_day b/merge_videos_by_day
index a6b509b..9b8e2d7 100755
--- a/merge_videos_by_day
+++ b/merge_videos_by_day
@@ -9,7 +9,7 @@ use File::stat;
 use Time::localtime;
 
 # Early command line options processing
-getopts("rh:tvs:");
+getopts("qrh:tvs:");
 my $srcpathname = $opt_s;
 
 ####################################################################################################
@@ -17,12 +17,16 @@ my $srcpathname = $opt_s;
 my $compute_host = "pippin.pippins.net"; # I need this since this script is run from a virtual machine 
 my $use_compute_host = 1; # Set to 1 to use a remote compute host to run the mkvmerge command. Set to 0 to use the local host to run it.
 my $make_mkv = "/naspool/videos/bin/make_mkv"; # Update this to be the path to the make_mkv script
+my $requantize_input_video=1; # This will dramatically decrease the size of the video with minimal compute processing requirements.
 my $owner = "ajp"; # The owner of the files after they are moved
 my $group = "pip"; # The group of the files after they are moved
 my $mode = "664"; # The mode to set on each file after they are moved
 my $video_suffix = "000"; # What number to start with when adding an incrementing suffix to the end of the video clip to avoid name collisons
 my $video_title_prefix = "HomeVideos:"; # What text to put on the front of the title for the merged video being created
 my $find_cmd = "find \"$srcpathname/\" -iregex \".*\.mov\" -o -iregex \".*\.3gp\" -o -iregex \".*\.mp4\" -o -iregex \".*\.mts\"";
+my $handbrake='HandBrakeCLI';
+my $interlaced_requantize_quality=0.85;
+my $progressive_requantize_quality=0.7;
 ####################################################################################################
 
 sub usage {
@@ -32,12 +36,13 @@ sub usage {
     print "   -v                    verbose mode; print extra information about what is being found/merged\n";
     print "   -t                    test mode; print what will happen, but don't do anything\n";
     print "   -r                    remove merged video clips; after a merge, remove the individual video files that were merged\n";
+    print "   -q                    Requantize MTS input videos to decrease output video size (requires HandBrakeCLI)\n";
     return 1;
 }
 if(defined $opt_h) { usage(); exit 1; }
 
 # Sanity checks
-if(! -x $make_mkv) { die "-E- Unable to find required script: make_mkv\n"; }
+if(defined $opt_q && !$use_compute_host) { die "-E- Unable to find required program: handbrake\n"; }
 if(! -d $srcpathname) { &usage; print "-E- Can't find srcpath: $srcpathname\n"; exit 1; }
 if(defined $opt_h) { $compute_host = $opt_h; }
 
@@ -135,20 +140,27 @@ foreach $file (sort `$find_cmd`) {
     push(@{$videos{"$srcext"}{"$dstfile"}}, "\"$srcdir/$srcfile\"");
 }
 
-# Only merge the videos if there is more than 1 video to merge on a given day for a given ext
+# For single videos, with the re-quantize option given, rename the destination file to mp4
 foreach $ext (sort keys %videos) {
-    foreach $video (sort keys %{$videos{$ext}}) {
+    foreach $video (sort keys %{$videos{$ext}}) {	
+	# Get a count of the number of videos for this date
+	# If we only have a single video, its extension will actually be mp4 if opt_q is specified
 	my $num_videos = $#{$videos{$ext}{$video}} + 1;
-	if($num_videos <= 1) {
-	    delete $videos{$ext}{$video};
-	    next;
+	if((defined $opt_q) && ($num_videos <= 1)) {
+	    if($video =~ /(.*?)\.(\d+)\.(\w+)$/) {
+		$dstfile = $1;
+		$dstnum = $2;
+		$new_dstvideo = "$1.$2.mp4";
+		$videos{$ext}{$new_dstvideo} = $videos{$ext}{$video}; # make a new dst video entry with the src video being the same
+		delete $videos{$ext}{$video}; # delete the old destination video from the hash
+	    }
 	}
     }
 }
-    
+
 # Check for duplicate filenames in the dstfiles being created for other exts
 foreach $ext (sort keys %videos) {
-    foreach $video (sort keys %{$videos{$ext}}) {
+    foreach $video (sort keys %{$videos{$ext}}) {	
 	# Make sure this video name is not in use as a destination for any other ext
 	foreach $checkext (sort keys %videos) {
 	    if($checkext eq $ext) { next; }
@@ -171,6 +183,71 @@ foreach $ext (sort keys %videos) {
     }
 }
 
+# Only merge the videos if there is more than 1 video to merge on a given day for a given ext
+# If there is only 1 video for a given day for a given ext, re-quantize it here if that option was given
+foreach $ext (sort keys %videos) {
+    foreach $video (sort keys %{$videos{$ext}}) {
+	
+	# Get a count of the number of videos for this date
+	my $num_videos = $#{$videos{$ext}{$video}} + 1;
+
+	# Process any single videos now
+	if($num_videos <= 1) {
+
+	    # Store the srcvideo name
+	    my $srcvideo = $videos{$ext}{$video}[0];
+	    my $pwd = `pwd`; chomp($pwd);
+
+	    # Make a note if this video is interlaced or not
+	    my $ffmpeg_cmd = "";
+	    if($use_compute_host) { $ffmpeg_cmd .= "ssh $compute_host 'cd \"$pwd\";"; }
+	    $ffmpeg_cmd .= "$ffmpeg -i $srcvideo 2>&1 | grep -q \"frame rate differs\"";
+	    if($use_compute_host) { $ffmpeg_cmd .= "'"; }
+	    my $progressive = system('$ffmpeg_cmd');
+	    if(!$progressive) { print "   Detected interlaced video content: $srcvideo\n"; }
+	    
+	    # Re-quantize the input video to reduce the resulting output filesize
+	    # This also gives us a chance to deinterlace the video as well
+	    # Only do this for .MTS videos
+	    if((defined $opt_q) && ($ext =~ /mts/i)) {
+		
+		# Set our requantize factor accordingly
+		my $requantize_option = "";
+		if(!$progressive) { $requantize_option = "-q $interlaced_requantize_quality"; }
+		else { $requantize_option = "-q $progressive_requantize_quality"; }
+		
+		# Set our de-interlace option accordingly
+		my $deinterlace_option = "";
+		if(!$progressive) { $deinterlace_option = "-d"; }
+		
+		# Use HandBrake to requantize/deinterlace the input video
+		print "   Re-quantizing input video content: $video\n";
+		my $handbrake_cmd = "";
+		if($use_compute_host) { $handbrake_cmd .= "ssh $compute_host 'cd \"$pwd\";"; }
+		$handbrake_cmd .= "$handbrake $deinterlace_option $requantize_option -E ac3 -i $srcvideo -o \"$video\" > /dev/null 2>&1";
+		if($use_compute_host) { $handbrake_cmd .= "'"; }
+		if(! defined $opt_t) { 
+		    my $errno = system("$handbrake_cmd");
+		    $errno = $errno >> 8;
+		    if($errno > 1) {
+			unlink "$video";
+			die "-E- handbrake encountered some errors with exit code $errno\n";
+		    } else {
+			# Remove the original srcvideo since we created a new version of it that we are going to keep instead
+			if(defined $opt_r) { 
+			    system("rm -f $srcvideo\n");
+			}
+		    }
+		}
+	    }
+
+	    # Remove the video from the array since we already processed it above
+	    delete $videos{$ext}{$video};
+	    next;
+	}
+    }
+}
+
 # Tell the user which videos we are going to merge
 foreach $ext (sort keys %videos) {
     foreach $video (sort keys %{$videos{$ext}}) {
@@ -187,10 +264,6 @@ foreach $ext (sort keys %videos) {
 
 	my $videos = join(',', @{$videos{$ext}{$video}});
 	
-	# Only merge the videos if there is more than 1
-	#my $num_videos = $#{$videos{$ext}{$video}} + 1;
-	#if($num_videos <= 1) { next; }
-	
 	if($video =~ /(\d+)-(\d+)-(\d+)/) {
 	    $year = $1;
 	    $month = $2;
@@ -201,6 +274,7 @@ foreach $ext (sort keys %videos) {
 	my $cmd = "";
 	if($use_compute_host) { $cmd .= "ssh $compute_host 'cd \"$pwd\";"; }
 	$cmd .= "$make_mkv -t \"$video_title_prefix $year-$month-$day\" -o \"$video\" -i $videos";
+	if($requantize_input_video) { $cmd .= ' -q'; }
 	if($use_compute_host) { $cmd .= "'"; }
 	if(defined $opt_t) {
 	    print "\n-> Creating \"$video\"\n";
diff --git a/mkv2mp4 b/mkv2mp4
index afbd0a8..2d9d873 100755
--- a/mkv2mp4
+++ b/mkv2mp4
@@ -1,8 +1,14 @@
 #!/bin/bash
 
 INPUT=$1
-FILENAME=${INPUT%%.*}
-OUTPUT="$FILENAME.mp4"
+OUTPUT_DIR=$2
+
+if [ -z "$OUTPUT_DIR" ]; then
+    OUTPUT_DIR="."
+fi
+
+FILENAME=`basename ${INPUT%%.*}`
+OUTPUT="$OUTPUT_DIR/$FILENAME.mp4"
 
 echo "-> Converting $INPUT to $OUTPUT file"
 
@@ -11,6 +17,9 @@ if [[ -e "$OUTPUT" ]]; then
   exit 1
 fi
 
+# 720p
+# HandBrakeCLI -d -b 3000 -q 0.8 -w 1280 -i /naspool/videos/HomeVideos/2011/2011-12-01.000.mkv -o 2011-12-01.mp4
+
 ffmpeg -i $INPUT \
   -b:v 3000k \
   -vcodec libx264 \
diff --git a/mkv_extract_chapter b/mkv_extract_chapter
index b70a750..d8825ae 100755
--- a/mkv_extract_chapter
+++ b/mkv_extract_chapter
@@ -73,7 +73,7 @@ sub usage {
 # SUBROUTINES
 
 sub detect_ext {
-    my ($ffmpeg_info, $interlaced) = @_;
+    my ($ffmpeg_info, $progressive) = @_;
 
     my $h264 = 0;
     my $h264_high = 0;
@@ -94,9 +94,9 @@ sub detect_ext {
     # Quicktime/MOV
     if($h264 && $pcm_s16le) { return "mov"; }
     # MTS
-    if($h264_high && $ac3) { if($interlaced) { return "mkv"; } else { return "mp4"; } }
+    if($h264_high && $ac3) { if(!$progressive) { return "mkv"; } else { return "mp4"; } }
     # 3GP/MP4
-    if($h264 && $aac) { if($interlaced) { return "mkv"; } else { return "mp4"; } }
+    if($h264 && $aac) { if(!$progressive) { return "mkv"; } else { return "mp4"; } }
 
     return "UNKNOWN";
 }
@@ -118,8 +118,8 @@ my $all_chapters = 0;
 if($opt_c =~ /all/) { $all_chapters = 1; }
 print "-> Extracting the following chapters from $opt_i: @chapters\n";
 
-my $interlaced = system('ffmpeg -i "$opt_i" 2>&1 | grep -q "frame rate differs"');
-if($interlaced != 0) {
+my $progressive = system('ffmpeg -i "$opt_i" 2>&1 | grep -q "frame rate differs"');
+if(!$progressive) {
     print "   Detected interlaced video content\n";
 } else {
     print "   Detected progressive video content\n";
@@ -139,7 +139,7 @@ foreach $line (@ffmpeg_info) {
 	if($all_chapters || &export_chapter(\@chapters,$chapter)) {
 	    $ext = "UNKNOWN"; 
 	    if(defined $opt_e) { $ext = $opt_e; }
-	    else { $ext = &detect_ext(\@ffmpeg_info, $interlaced); } 
+	    else { $ext = &detect_ext(\@ffmpeg_info, $progressive); } 
 	    if($ext =~ /UNKNOWN/) { die "-E- Unable to determine the file type/extension to use for the output videos. Specify it with the -e <ext> option.\n"; }
 	    $dstfile = $opt_o . ".c" . sprintf("%03d",$chapter) . "." . $ext;
 	    print "-> Exporting $chapter to $dstfile:\n";
@@ -153,7 +153,7 @@ foreach $line (@ffmpeg_info) {
 		$errno = $errno >> 8;
 		if($errno > 0) { die "-E- ffmpeg encountered some errors with exit code $errno\n"; }
 	    }
-	    if($interlaced && $ext =~ /mkv/i) {
+	    if(!$progressive && $ext =~ /mkv/i) {
 		$ext = "mp4";
 		$srcfile = $dstfile;
 		$dstfile = $dstfile = $opt_o . ".c" . sprintf("%03d",$chapter) . "." . $ext;
diff --git a/mts2mpg b/mts2mpg
index e3e4962..c29e3f3 100755
--- a/mts2mpg
+++ b/mts2mpg
@@ -9,8 +9,8 @@ foreach(@files)
 
   # Find out if content is interlaced or not
   $deint_option = "-deinterlace";
-  $interlaced = system('ffmpeg -i "$_.MTS" 2>&1 | grep -q "frame rate differs"');
-  if($interlaced != 0) { 
+  $progressive = system('ffmpeg -i "$_.MTS" 2>&1 | grep -q "frame rate differs"');
+  if(!$progressive) { 
     print "   Detected interlaced video content\n";
   } else { 
     print "   Detected progressive video content\n";
diff --git a/organize_videos b/organize_videos
index 86047a1..e8b02db 100755
--- a/organize_videos
+++ b/organize_videos
@@ -14,7 +14,7 @@ use File::Pid;
 my $srcpathname = "/naspool/pictures/New Photos"; # Path to look for videos to move from
 my $dstpathname = "/naspool/videos/HomeVideos"; # Path to move the videos to
 my $merge_videos_by_day = "/naspool/videos/bin/merge_videos_by_day";
-my $minage = "+30"; # Video file creation dates must not have changed in the last X minutes to process any of the video files
+my $minage = "+0"; # Video file creation dates must not have changed in the last X minutes to process any of the video files
 my $owner = "ajp"; # The owner of the files after they are moved
 my $group = "pip"; # The group of the files after they are moved
 my $mode = "664"; # The mode to set on each file after they are moved
@@ -142,7 +142,7 @@ my $errno = 0;
 my $merge_opts = "";
 if(defined $opt_t) { $merge_opts .= "-t "; }
 if(defined $opt_r) { $merge_opts .= "-r "; }
-$errno=system("$merge_videos_by_day -s \"$srcpathname\" $merge_opts");
+$errno=system("$merge_videos_by_day -q -s \"$srcpathname\" $merge_opts");
 $errno = $errno >> 8;
 if($errno) { die "-E- $merge_videos_by_day encountered some errors with exit code $errno\n"; }
 
@@ -230,7 +230,8 @@ foreach $file (`$find_cmd`) {
 	  if(! defined $opt_t) { 
 	      $tmpdir=`tempfile`;
 	      system("rm $tmpdir");
-	      system("mv \"$srcdir\" $tmpdir > /dev/null 2>/dev/null");
+	      system("cp -R \"$srcdir\" $tmpdir > /dev/null 2>/dev/null");
+	      system("rm -rf \"$srcdir\"");
 	      print "-> Moved empty subdir $srcdir to $tmpdir\n" if($opt_v);
 	  }
       }