From d0bb8d7ba0914fb2e31422aa1207ba6eaf490deb Mon Sep 17 00:00:00 2001 From: "Alan J. Pippin" Date: Sat, 24 Dec 2011 15:37:31 -0700 Subject: [PATCH] Major changes to support interlaced/progressive content Major changes to support re-quantization to significantly downsize large HD video streams with little impact to quality. --- make_mkv | 62 +++++++++++++++++++++------ merge_videos_by_day | 100 ++++++++++++++++++++++++++++++++++++++------ mkv2mp4 | 13 +++++- mkv_extract_chapter | 14 +++---- mts2mpg | 4 +- organize_videos | 7 ++-- 6 files changed, 161 insertions(+), 39 deletions(-) diff --git a/make_mkv b/make_mkv index 04556e5..c4f3707 100755 --- a/make_mkv +++ b/make_mkv @@ -18,22 +18,23 @@ use DateTime::Format::Duration; #################################################################################################### # Configuration parameters - CHANGE THESE TO SUITE YOUR NEEDS -my $mkvmerge=`which mkvmerge`; chomp($mkvmerge); -my $ffmpeg=`which ffmpeg`; chomp($ffmpeg); +my $mkvmerge='mkvmerge'; +my $ffmpeg='ffmpeg'; +my $handbrake='HandBrakeCLI'; my $tmpfile = `tempfile`; chomp($tmpfile); my $chapter_file = $tmpfile; my $input_file_options = "-S"; my $output_file_options = "--chapters $chapter_file --compression -1:none"; my $timezone = `cat /etc/timezone`; chomp($timezone); +my $interlaced_requantize_quality=0.85; +my $progressive_requantize_quality=0.7; #################################################################################################### #################################################################################################### # Command Line Options -getopts("st:o:i:h"); +getopts("sqt:o:i:h"); -if(! -x $mkvmerge) { die "-E- Unable to find required program: mkvmerge\n"; } -if(! -x $ffmpeg) { die "-E- Unable to find required program: ffmpeg\n"; } if(! defined $opt_t) { &usage(); die "-E- Missing required title: -t \n"; } if(! defined $opt_o) { &usage(); die "-E- Missing required argument output video names: -o <output.mkv>\n"; } if(! defined $opt_i) { &usage(); die "-E- Missing required argument input video names: -i <input,input,...>\n"; } @@ -43,6 +44,7 @@ sub usage { print " -t <title> Sets the general title for the output video file\n"; print " -o <output.mkv> Sets the name of the output mkv file\n"; print " -i <input,input,...> Sets the name of the input files to make into an mkv file\n"; + print " -q Requantize MTS input videos to decrease output video size (requires HandBrakeCLI)\n"; print " -s Simulate mode. Don't actually make the video, but tell us what you will do\n"; print "\n"; return 1; @@ -111,22 +113,24 @@ my $cmd = "$mkvmerge --title \"$opt_t\" $output_file_options -o \"$opt_o\""; my $video_count = 0; my @video_tmp_files; foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { + # Make a note of the video extension + my $video_ext = $video; + $video_ext =~ s/.*\.(\S+)$/$1/; + # Detect if the input file is interlaced or not. # There is a bug in mkvmerge 5.0.1 and earlier where if the video content is 1080i, it doesn't mux it properly, and it stutters. # The quick solution to this issue is to run the interlaced video file through ffmpeg and tell it to copy the video/audio streams to a mkv container. # We will then merge this temporarily created mkv container into the final mkv container instead of the original interlaced video. # http://ubuntuforums.org/showthread.php?t=1627194 # http://forum.doom9.org/showthread.php?t=155732&page=31 - my $interlaced = system('ffmpeg -i "$video" 2>&1 | grep -q "frame rate differs"'); - if($interlaced != 0) { + my $progressive = system('$ffmpeg -i "$video" 2>&1 | grep -q "frame rate differs"'); + if(!$progressive) { my $video_mkv = $video; - my $video_ext = $video; - $video_ext =~ s/.*\.(\S+)$/$1/; print " Detected interlaced video content: $video\n"; if($video_ext !~ /mkv/i) { - $video_mkv =~ s/\.[^.]*$//; $video_mkv .= ".mkv"; + $video_mkv =~ s/\.[^.]*$//; $video_mkv .= ".ffmpeg.mkv"; print " Re-muxing the interlaced video content as an mkv file: $video_mkv\n"; - my $make_mkv_cmd = "ffmpeg -i \"$video\" -scodec copy -acodec copy -vcodec copy -f matroska \"$video_mkv\" > /dev/null 2>&1"; + my $make_mkv_cmd = "$ffmpeg -y -i \"$video\" -scodec copy -acodec copy -vcodec copy -f matroska \"$video_mkv\" > /dev/null 2>&1"; if(! defined $opt_s) { my $errno = system("$make_mkv_cmd"); $errno = $errno >> 8; @@ -143,7 +147,41 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { } else { print " Detected progressive video content: $video\n"; } - # Create our intput file command line options for this video + + # Re-quantize the input video to reduce the resulting output filesize + # This also gives us a chance to deinterlace the video as well + # Only do this for .MTS videos + if((defined $opt_q) && ($video_ext =~ /mts/i)) { + + # Set our requantize factor accordingly + my $requantize_option = ""; + if(!$progressive) { $requantize_option = "-q $interlaced_requantize_quality"; } + else { $requantize_option = "-q $progressive_requantize_quality"; } + + # Set our de-interlace option accordingly + my $deinterlace_option = ""; + if(!$progressive) { $deinterlace_option = "-d"; } + + # Use HandBrake to requantize/deinterlace the input video + my $video_mp4 = $video; $video_mp4 =~ s/\.[^.]*$//; $video_mp4 .= ".hb.mp4"; + print " Re-quantizing input video content: $video_mp4\n"; + my $handbrake_cmd = "$handbrake $deinterlace_option $requantize_option -E ac3 -i \"$video\" -o \"$video_mp4\" > /dev/null 2>&1"; + if(! defined $opt_s) { + my $errno = system("$handbrake_cmd"); + $errno = $errno >> 8; + if($errno > 1) { + unlink "$video_mp4"; + die "-E- handbrake encountered some errors with exit code $errno\n"; + } + } + + # Push the name of our intermediate video file onto a list of files to delete on exit + push(@video_tmp_files, "$video_mp4"); + # Update the name of our video file to equal the name of our new intermediate video file name + $video = $video_mp4; + } + + # Create our input file command line options for this video if($video_count == 0) { $cmd .= " $input_file_options \"$video\""; } else { diff --git a/merge_videos_by_day b/merge_videos_by_day index a6b509b..9b8e2d7 100755 --- a/merge_videos_by_day +++ b/merge_videos_by_day @@ -9,7 +9,7 @@ use File::stat; use Time::localtime; # Early command line options processing -getopts("rh:tvs:"); +getopts("qrh:tvs:"); my $srcpathname = $opt_s; #################################################################################################### @@ -17,12 +17,16 @@ my $srcpathname = $opt_s; my $compute_host = "pippin.pippins.net"; # I need this since this script is run from a virtual machine my $use_compute_host = 1; # Set to 1 to use a remote compute host to run the mkvmerge command. Set to 0 to use the local host to run it. my $make_mkv = "/naspool/videos/bin/make_mkv"; # Update this to be the path to the make_mkv script +my $requantize_input_video=1; # This will dramatically decrease the size of the video with minimal compute processing requirements. my $owner = "ajp"; # The owner of the files after they are moved my $group = "pip"; # The group of the files after they are moved my $mode = "664"; # The mode to set on each file after they are moved my $video_suffix = "000"; # What number to start with when adding an incrementing suffix to the end of the video clip to avoid name collisons my $video_title_prefix = "HomeVideos:"; # What text to put on the front of the title for the merged video being created my $find_cmd = "find \"$srcpathname/\" -iregex \".*\.mov\" -o -iregex \".*\.3gp\" -o -iregex \".*\.mp4\" -o -iregex \".*\.mts\""; +my $handbrake='HandBrakeCLI'; +my $interlaced_requantize_quality=0.85; +my $progressive_requantize_quality=0.7; #################################################################################################### sub usage { @@ -32,12 +36,13 @@ sub usage { print " -v verbose mode; print extra information about what is being found/merged\n"; print " -t test mode; print what will happen, but don't do anything\n"; print " -r remove merged video clips; after a merge, remove the individual video files that were merged\n"; + print " -q Requantize MTS input videos to decrease output video size (requires HandBrakeCLI)\n"; return 1; } if(defined $opt_h) { usage(); exit 1; } # Sanity checks -if(! -x $make_mkv) { die "-E- Unable to find required script: make_mkv\n"; } +if(defined $opt_q && !$use_compute_host) { die "-E- Unable to find required program: handbrake\n"; } if(! -d $srcpathname) { &usage; print "-E- Can't find srcpath: $srcpathname\n"; exit 1; } if(defined $opt_h) { $compute_host = $opt_h; } @@ -135,20 +140,27 @@ foreach $file (sort `$find_cmd`) { push(@{$videos{"$srcext"}{"$dstfile"}}, "\"$srcdir/$srcfile\""); } -# Only merge the videos if there is more than 1 video to merge on a given day for a given ext +# For single videos, with the re-quantize option given, rename the destination file to mp4 foreach $ext (sort keys %videos) { - foreach $video (sort keys %{$videos{$ext}}) { + foreach $video (sort keys %{$videos{$ext}}) { + # Get a count of the number of videos for this date + # If we only have a single video, its extension will actually be mp4 if opt_q is specified my $num_videos = $#{$videos{$ext}{$video}} + 1; - if($num_videos <= 1) { - delete $videos{$ext}{$video}; - next; + if((defined $opt_q) && ($num_videos <= 1)) { + if($video =~ /(.*?)\.(\d+)\.(\w+)$/) { + $dstfile = $1; + $dstnum = $2; + $new_dstvideo = "$1.$2.mp4"; + $videos{$ext}{$new_dstvideo} = $videos{$ext}{$video}; # make a new dst video entry with the src video being the same + delete $videos{$ext}{$video}; # delete the old destination video from the hash + } } } } - + # Check for duplicate filenames in the dstfiles being created for other exts foreach $ext (sort keys %videos) { - foreach $video (sort keys %{$videos{$ext}}) { + foreach $video (sort keys %{$videos{$ext}}) { # Make sure this video name is not in use as a destination for any other ext foreach $checkext (sort keys %videos) { if($checkext eq $ext) { next; } @@ -171,6 +183,71 @@ foreach $ext (sort keys %videos) { } } +# Only merge the videos if there is more than 1 video to merge on a given day for a given ext +# If there is only 1 video for a given day for a given ext, re-quantize it here if that option was given +foreach $ext (sort keys %videos) { + foreach $video (sort keys %{$videos{$ext}}) { + + # Get a count of the number of videos for this date + my $num_videos = $#{$videos{$ext}{$video}} + 1; + + # Process any single videos now + if($num_videos <= 1) { + + # Store the srcvideo name + my $srcvideo = $videos{$ext}{$video}[0]; + my $pwd = `pwd`; chomp($pwd); + + # Make a note if this video is interlaced or not + my $ffmpeg_cmd = ""; + if($use_compute_host) { $ffmpeg_cmd .= "ssh $compute_host 'cd \"$pwd\";"; } + $ffmpeg_cmd .= "$ffmpeg -i $srcvideo 2>&1 | grep -q \"frame rate differs\""; + if($use_compute_host) { $ffmpeg_cmd .= "'"; } + my $progressive = system('$ffmpeg_cmd'); + if(!$progressive) { print " Detected interlaced video content: $srcvideo\n"; } + + # Re-quantize the input video to reduce the resulting output filesize + # This also gives us a chance to deinterlace the video as well + # Only do this for .MTS videos + if((defined $opt_q) && ($ext =~ /mts/i)) { + + # Set our requantize factor accordingly + my $requantize_option = ""; + if(!$progressive) { $requantize_option = "-q $interlaced_requantize_quality"; } + else { $requantize_option = "-q $progressive_requantize_quality"; } + + # Set our de-interlace option accordingly + my $deinterlace_option = ""; + if(!$progressive) { $deinterlace_option = "-d"; } + + # Use HandBrake to requantize/deinterlace the input video + print " Re-quantizing input video content: $video\n"; + my $handbrake_cmd = ""; + if($use_compute_host) { $handbrake_cmd .= "ssh $compute_host 'cd \"$pwd\";"; } + $handbrake_cmd .= "$handbrake $deinterlace_option $requantize_option -E ac3 -i $srcvideo -o \"$video\" > /dev/null 2>&1"; + if($use_compute_host) { $handbrake_cmd .= "'"; } + if(! defined $opt_t) { + my $errno = system("$handbrake_cmd"); + $errno = $errno >> 8; + if($errno > 1) { + unlink "$video"; + die "-E- handbrake encountered some errors with exit code $errno\n"; + } else { + # Remove the original srcvideo since we created a new version of it that we are going to keep instead + if(defined $opt_r) { + system("rm -f $srcvideo\n"); + } + } + } + } + + # Remove the video from the array since we already processed it above + delete $videos{$ext}{$video}; + next; + } + } +} + # Tell the user which videos we are going to merge foreach $ext (sort keys %videos) { foreach $video (sort keys %{$videos{$ext}}) { @@ -187,10 +264,6 @@ foreach $ext (sort keys %videos) { my $videos = join(',', @{$videos{$ext}{$video}}); - # Only merge the videos if there is more than 1 - #my $num_videos = $#{$videos{$ext}{$video}} + 1; - #if($num_videos <= 1) { next; } - if($video =~ /(\d+)-(\d+)-(\d+)/) { $year = $1; $month = $2; @@ -201,6 +274,7 @@ foreach $ext (sort keys %videos) { my $cmd = ""; if($use_compute_host) { $cmd .= "ssh $compute_host 'cd \"$pwd\";"; } $cmd .= "$make_mkv -t \"$video_title_prefix $year-$month-$day\" -o \"$video\" -i $videos"; + if($requantize_input_video) { $cmd .= ' -q'; } if($use_compute_host) { $cmd .= "'"; } if(defined $opt_t) { print "\n-> Creating \"$video\"\n"; diff --git a/mkv2mp4 b/mkv2mp4 index afbd0a8..2d9d873 100755 --- a/mkv2mp4 +++ b/mkv2mp4 @@ -1,8 +1,14 @@ #!/bin/bash INPUT=$1 -FILENAME=${INPUT%%.*} -OUTPUT="$FILENAME.mp4" +OUTPUT_DIR=$2 + +if [ -z "$OUTPUT_DIR" ]; then + OUTPUT_DIR="." +fi + +FILENAME=`basename ${INPUT%%.*}` +OUTPUT="$OUTPUT_DIR/$FILENAME.mp4" echo "-> Converting $INPUT to $OUTPUT file" @@ -11,6 +17,9 @@ if [[ -e "$OUTPUT" ]]; then exit 1 fi +# 720p +# HandBrakeCLI -d -b 3000 -q 0.8 -w 1280 -i /naspool/videos/HomeVideos/2011/2011-12-01.000.mkv -o 2011-12-01.mp4 + ffmpeg -i $INPUT \ -b:v 3000k \ -vcodec libx264 \ diff --git a/mkv_extract_chapter b/mkv_extract_chapter index b70a750..d8825ae 100755 --- a/mkv_extract_chapter +++ b/mkv_extract_chapter @@ -73,7 +73,7 @@ sub usage { # SUBROUTINES sub detect_ext { - my ($ffmpeg_info, $interlaced) = @_; + my ($ffmpeg_info, $progressive) = @_; my $h264 = 0; my $h264_high = 0; @@ -94,9 +94,9 @@ sub detect_ext { # Quicktime/MOV if($h264 && $pcm_s16le) { return "mov"; } # MTS - if($h264_high && $ac3) { if($interlaced) { return "mkv"; } else { return "mp4"; } } + if($h264_high && $ac3) { if(!$progressive) { return "mkv"; } else { return "mp4"; } } # 3GP/MP4 - if($h264 && $aac) { if($interlaced) { return "mkv"; } else { return "mp4"; } } + if($h264 && $aac) { if(!$progressive) { return "mkv"; } else { return "mp4"; } } return "UNKNOWN"; } @@ -118,8 +118,8 @@ my $all_chapters = 0; if($opt_c =~ /all/) { $all_chapters = 1; } print "-> Extracting the following chapters from $opt_i: @chapters\n"; -my $interlaced = system('ffmpeg -i "$opt_i" 2>&1 | grep -q "frame rate differs"'); -if($interlaced != 0) { +my $progressive = system('ffmpeg -i "$opt_i" 2>&1 | grep -q "frame rate differs"'); +if(!$progressive) { print " Detected interlaced video content\n"; } else { print " Detected progressive video content\n"; @@ -139,7 +139,7 @@ foreach $line (@ffmpeg_info) { if($all_chapters || &export_chapter(\@chapters,$chapter)) { $ext = "UNKNOWN"; if(defined $opt_e) { $ext = $opt_e; } - else { $ext = &detect_ext(\@ffmpeg_info, $interlaced); } + else { $ext = &detect_ext(\@ffmpeg_info, $progressive); } if($ext =~ /UNKNOWN/) { die "-E- Unable to determine the file type/extension to use for the output videos. Specify it with the -e <ext> option.\n"; } $dstfile = $opt_o . ".c" . sprintf("%03d",$chapter) . "." . $ext; print "-> Exporting $chapter to $dstfile:\n"; @@ -153,7 +153,7 @@ foreach $line (@ffmpeg_info) { $errno = $errno >> 8; if($errno > 0) { die "-E- ffmpeg encountered some errors with exit code $errno\n"; } } - if($interlaced && $ext =~ /mkv/i) { + if(!$progressive && $ext =~ /mkv/i) { $ext = "mp4"; $srcfile = $dstfile; $dstfile = $dstfile = $opt_o . ".c" . sprintf("%03d",$chapter) . "." . $ext; diff --git a/mts2mpg b/mts2mpg index e3e4962..c29e3f3 100755 --- a/mts2mpg +++ b/mts2mpg @@ -9,8 +9,8 @@ foreach(@files) # Find out if content is interlaced or not $deint_option = "-deinterlace"; - $interlaced = system('ffmpeg -i "$_.MTS" 2>&1 | grep -q "frame rate differs"'); - if($interlaced != 0) { + $progressive = system('ffmpeg -i "$_.MTS" 2>&1 | grep -q "frame rate differs"'); + if(!$progressive) { print " Detected interlaced video content\n"; } else { print " Detected progressive video content\n"; diff --git a/organize_videos b/organize_videos index 86047a1..e8b02db 100755 --- a/organize_videos +++ b/organize_videos @@ -14,7 +14,7 @@ use File::Pid; my $srcpathname = "/naspool/pictures/New Photos"; # Path to look for videos to move from my $dstpathname = "/naspool/videos/HomeVideos"; # Path to move the videos to my $merge_videos_by_day = "/naspool/videos/bin/merge_videos_by_day"; -my $minage = "+30"; # Video file creation dates must not have changed in the last X minutes to process any of the video files +my $minage = "+0"; # Video file creation dates must not have changed in the last X minutes to process any of the video files my $owner = "ajp"; # The owner of the files after they are moved my $group = "pip"; # The group of the files after they are moved my $mode = "664"; # The mode to set on each file after they are moved @@ -142,7 +142,7 @@ my $errno = 0; my $merge_opts = ""; if(defined $opt_t) { $merge_opts .= "-t "; } if(defined $opt_r) { $merge_opts .= "-r "; } -$errno=system("$merge_videos_by_day -s \"$srcpathname\" $merge_opts"); +$errno=system("$merge_videos_by_day -q -s \"$srcpathname\" $merge_opts"); $errno = $errno >> 8; if($errno) { die "-E- $merge_videos_by_day encountered some errors with exit code $errno\n"; } @@ -230,7 +230,8 @@ foreach $file (`$find_cmd`) { if(! defined $opt_t) { $tmpdir=`tempfile`; system("rm $tmpdir"); - system("mv \"$srcdir\" $tmpdir > /dev/null 2>/dev/null"); + system("cp -R \"$srcdir\" $tmpdir > /dev/null 2>/dev/null"); + system("rm -rf \"$srcdir\""); print "-> Moved empty subdir $srcdir to $tmpdir\n" if($opt_v); } } -- 2.34.1