Major changes to support re-quantization to significantly downsize large HD video streams with little impact to quality.
####################################################################################################
# Configuration parameters - CHANGE THESE TO SUITE YOUR NEEDS
-my $mkvmerge=`which mkvmerge`; chomp($mkvmerge);
-my $ffmpeg=`which ffmpeg`; chomp($ffmpeg);
+my $mkvmerge='mkvmerge';
+my $ffmpeg='ffmpeg';
+my $handbrake='HandBrakeCLI';
my $tmpfile = `tempfile`; chomp($tmpfile);
my $chapter_file = $tmpfile;
my $input_file_options = "-S";
my $output_file_options = "--chapters $chapter_file --compression -1:none";
my $timezone = `cat /etc/timezone`; chomp($timezone);
+my $interlaced_requantize_quality=0.85;
+my $progressive_requantize_quality=0.7;
####################################################################################################
####################################################################################################
# Command Line Options
-getopts("st:o:i:h");
+getopts("sqt:o:i:h");
-if(! -x $mkvmerge) { die "-E- Unable to find required program: mkvmerge\n"; }
-if(! -x $ffmpeg) { die "-E- Unable to find required program: ffmpeg\n"; }
if(! defined $opt_t) { &usage(); die "-E- Missing required title: -t <title>\n"; }
if(! defined $opt_o) { &usage(); die "-E- Missing required argument output video names: -o <output.mkv>\n"; }
if(! defined $opt_i) { &usage(); die "-E- Missing required argument input video names: -i <input,input,...>\n"; }
print " -t <title> Sets the general title for the output video file\n";
print " -o <output.mkv> Sets the name of the output mkv file\n";
print " -i <input,input,...> Sets the name of the input files to make into an mkv file\n";
+ print " -q Requantize MTS input videos to decrease output video size (requires HandBrakeCLI)\n";
print " -s Simulate mode. Don't actually make the video, but tell us what you will do\n";
print "\n";
return 1;
my $video_count = 0;
my @video_tmp_files;
foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) {
+ # Make a note of the video extension
+ my $video_ext = $video;
+ $video_ext =~ s/.*\.(\S+)$/$1/;
+
# Detect if the input file is interlaced or not.
# There is a bug in mkvmerge 5.0.1 and earlier where if the video content is 1080i, it doesn't mux it properly, and it stutters.
# The quick solution to this issue is to run the interlaced video file through ffmpeg and tell it to copy the video/audio streams to a mkv container.
# We will then merge this temporarily created mkv container into the final mkv container instead of the original interlaced video.
# http://ubuntuforums.org/showthread.php?t=1627194
# http://forum.doom9.org/showthread.php?t=155732&page=31
- my $interlaced = system('ffmpeg -i "$video" 2>&1 | grep -q "frame rate differs"');
- if($interlaced != 0) {
+ my $progressive = system('$ffmpeg -i "$video" 2>&1 | grep -q "frame rate differs"');
+ if(!$progressive) {
my $video_mkv = $video;
- my $video_ext = $video;
- $video_ext =~ s/.*\.(\S+)$/$1/;
print " Detected interlaced video content: $video\n";
if($video_ext !~ /mkv/i) {
- $video_mkv =~ s/\.[^.]*$//; $video_mkv .= ".mkv";
+ $video_mkv =~ s/\.[^.]*$//; $video_mkv .= ".ffmpeg.mkv";
print " Re-muxing the interlaced video content as an mkv file: $video_mkv\n";
- my $make_mkv_cmd = "ffmpeg -i \"$video\" -scodec copy -acodec copy -vcodec copy -f matroska \"$video_mkv\" > /dev/null 2>&1";
+ my $make_mkv_cmd = "$ffmpeg -y -i \"$video\" -scodec copy -acodec copy -vcodec copy -f matroska \"$video_mkv\" > /dev/null 2>&1";
if(! defined $opt_s) {
my $errno = system("$make_mkv_cmd");
$errno = $errno >> 8;
} else {
print " Detected progressive video content: $video\n";
}
- # Create our intput file command line options for this video
+
+ # Re-quantize the input video to reduce the resulting output filesize
+ # This also gives us a chance to deinterlace the video as well
+ # Only do this for .MTS videos
+ if((defined $opt_q) && ($video_ext =~ /mts/i)) {
+
+ # Set our requantize factor accordingly
+ my $requantize_option = "";
+ if(!$progressive) { $requantize_option = "-q $interlaced_requantize_quality"; }
+ else { $requantize_option = "-q $progressive_requantize_quality"; }
+
+ # Set our de-interlace option accordingly
+ my $deinterlace_option = "";
+ if(!$progressive) { $deinterlace_option = "-d"; }
+
+ # Use HandBrake to requantize/deinterlace the input video
+ my $video_mp4 = $video; $video_mp4 =~ s/\.[^.]*$//; $video_mp4 .= ".hb.mp4";
+ print " Re-quantizing input video content: $video_mp4\n";
+ my $handbrake_cmd = "$handbrake $deinterlace_option $requantize_option -E ac3 -i \"$video\" -o \"$video_mp4\" > /dev/null 2>&1";
+ if(! defined $opt_s) {
+ my $errno = system("$handbrake_cmd");
+ $errno = $errno >> 8;
+ if($errno > 1) {
+ unlink "$video_mp4";
+ die "-E- handbrake encountered some errors with exit code $errno\n";
+ }
+ }
+
+ # Push the name of our intermediate video file onto a list of files to delete on exit
+ push(@video_tmp_files, "$video_mp4");
+ # Update the name of our video file to equal the name of our new intermediate video file name
+ $video = $video_mp4;
+ }
+
+ # Create our input file command line options for this video
if($video_count == 0) {
$cmd .= " $input_file_options \"$video\"";
} else {
use Time::localtime;
# Early command line options processing
-getopts("rh:tvs:");
+getopts("qrh:tvs:");
my $srcpathname = $opt_s;
####################################################################################################
my $compute_host = "pippin.pippins.net"; # I need this since this script is run from a virtual machine
my $use_compute_host = 1; # Set to 1 to use a remote compute host to run the mkvmerge command. Set to 0 to use the local host to run it.
my $make_mkv = "/naspool/videos/bin/make_mkv"; # Update this to be the path to the make_mkv script
+my $requantize_input_video=1; # This will dramatically decrease the size of the video with minimal compute processing requirements.
my $owner = "ajp"; # The owner of the files after they are moved
my $group = "pip"; # The group of the files after they are moved
my $mode = "664"; # The mode to set on each file after they are moved
my $video_suffix = "000"; # What number to start with when adding an incrementing suffix to the end of the video clip to avoid name collisons
my $video_title_prefix = "HomeVideos:"; # What text to put on the front of the title for the merged video being created
my $find_cmd = "find \"$srcpathname/\" -iregex \".*\.mov\" -o -iregex \".*\.3gp\" -o -iregex \".*\.mp4\" -o -iregex \".*\.mts\"";
+my $handbrake='HandBrakeCLI';
+my $interlaced_requantize_quality=0.85;
+my $progressive_requantize_quality=0.7;
####################################################################################################
sub usage {
print " -v verbose mode; print extra information about what is being found/merged\n";
print " -t test mode; print what will happen, but don't do anything\n";
print " -r remove merged video clips; after a merge, remove the individual video files that were merged\n";
+ print " -q Requantize MTS input videos to decrease output video size (requires HandBrakeCLI)\n";
return 1;
}
if(defined $opt_h) { usage(); exit 1; }
# Sanity checks
-if(! -x $make_mkv) { die "-E- Unable to find required script: make_mkv\n"; }
+if(defined $opt_q && !$use_compute_host) { die "-E- Unable to find required program: handbrake\n"; }
if(! -d $srcpathname) { &usage; print "-E- Can't find srcpath: $srcpathname\n"; exit 1; }
if(defined $opt_h) { $compute_host = $opt_h; }
push(@{$videos{"$srcext"}{"$dstfile"}}, "\"$srcdir/$srcfile\"");
}
-# Only merge the videos if there is more than 1 video to merge on a given day for a given ext
+# For single videos, with the re-quantize option given, rename the destination file to mp4
foreach $ext (sort keys %videos) {
- foreach $video (sort keys %{$videos{$ext}}) {
+ foreach $video (sort keys %{$videos{$ext}}) {
+ # Get a count of the number of videos for this date
+ # If we only have a single video, its extension will actually be mp4 if opt_q is specified
my $num_videos = $#{$videos{$ext}{$video}} + 1;
- if($num_videos <= 1) {
- delete $videos{$ext}{$video};
- next;
+ if((defined $opt_q) && ($num_videos <= 1)) {
+ if($video =~ /(.*?)\.(\d+)\.(\w+)$/) {
+ $dstfile = $1;
+ $dstnum = $2;
+ $new_dstvideo = "$1.$2.mp4";
+ $videos{$ext}{$new_dstvideo} = $videos{$ext}{$video}; # make a new dst video entry with the src video being the same
+ delete $videos{$ext}{$video}; # delete the old destination video from the hash
+ }
}
}
}
-
+
# Check for duplicate filenames in the dstfiles being created for other exts
foreach $ext (sort keys %videos) {
- foreach $video (sort keys %{$videos{$ext}}) {
+ foreach $video (sort keys %{$videos{$ext}}) {
# Make sure this video name is not in use as a destination for any other ext
foreach $checkext (sort keys %videos) {
if($checkext eq $ext) { next; }
}
}
+# Only merge the videos if there is more than 1 video to merge on a given day for a given ext
+# If there is only 1 video for a given day for a given ext, re-quantize it here if that option was given
+foreach $ext (sort keys %videos) {
+ foreach $video (sort keys %{$videos{$ext}}) {
+
+ # Get a count of the number of videos for this date
+ my $num_videos = $#{$videos{$ext}{$video}} + 1;
+
+ # Process any single videos now
+ if($num_videos <= 1) {
+
+ # Store the srcvideo name
+ my $srcvideo = $videos{$ext}{$video}[0];
+ my $pwd = `pwd`; chomp($pwd);
+
+ # Make a note if this video is interlaced or not
+ my $ffmpeg_cmd = "";
+ if($use_compute_host) { $ffmpeg_cmd .= "ssh $compute_host 'cd \"$pwd\";"; }
+ $ffmpeg_cmd .= "$ffmpeg -i $srcvideo 2>&1 | grep -q \"frame rate differs\"";
+ if($use_compute_host) { $ffmpeg_cmd .= "'"; }
+ my $progressive = system('$ffmpeg_cmd');
+ if(!$progressive) { print " Detected interlaced video content: $srcvideo\n"; }
+
+ # Re-quantize the input video to reduce the resulting output filesize
+ # This also gives us a chance to deinterlace the video as well
+ # Only do this for .MTS videos
+ if((defined $opt_q) && ($ext =~ /mts/i)) {
+
+ # Set our requantize factor accordingly
+ my $requantize_option = "";
+ if(!$progressive) { $requantize_option = "-q $interlaced_requantize_quality"; }
+ else { $requantize_option = "-q $progressive_requantize_quality"; }
+
+ # Set our de-interlace option accordingly
+ my $deinterlace_option = "";
+ if(!$progressive) { $deinterlace_option = "-d"; }
+
+ # Use HandBrake to requantize/deinterlace the input video
+ print " Re-quantizing input video content: $video\n";
+ my $handbrake_cmd = "";
+ if($use_compute_host) { $handbrake_cmd .= "ssh $compute_host 'cd \"$pwd\";"; }
+ $handbrake_cmd .= "$handbrake $deinterlace_option $requantize_option -E ac3 -i $srcvideo -o \"$video\" > /dev/null 2>&1";
+ if($use_compute_host) { $handbrake_cmd .= "'"; }
+ if(! defined $opt_t) {
+ my $errno = system("$handbrake_cmd");
+ $errno = $errno >> 8;
+ if($errno > 1) {
+ unlink "$video";
+ die "-E- handbrake encountered some errors with exit code $errno\n";
+ } else {
+ # Remove the original srcvideo since we created a new version of it that we are going to keep instead
+ if(defined $opt_r) {
+ system("rm -f $srcvideo\n");
+ }
+ }
+ }
+ }
+
+ # Remove the video from the array since we already processed it above
+ delete $videos{$ext}{$video};
+ next;
+ }
+ }
+}
+
# Tell the user which videos we are going to merge
foreach $ext (sort keys %videos) {
foreach $video (sort keys %{$videos{$ext}}) {
my $videos = join(',', @{$videos{$ext}{$video}});
- # Only merge the videos if there is more than 1
- #my $num_videos = $#{$videos{$ext}{$video}} + 1;
- #if($num_videos <= 1) { next; }
-
if($video =~ /(\d+)-(\d+)-(\d+)/) {
$year = $1;
$month = $2;
my $cmd = "";
if($use_compute_host) { $cmd .= "ssh $compute_host 'cd \"$pwd\";"; }
$cmd .= "$make_mkv -t \"$video_title_prefix $year-$month-$day\" -o \"$video\" -i $videos";
+ if($requantize_input_video) { $cmd .= ' -q'; }
if($use_compute_host) { $cmd .= "'"; }
if(defined $opt_t) {
print "\n-> Creating \"$video\"\n";
#!/bin/bash
INPUT=$1
-FILENAME=${INPUT%%.*}
-OUTPUT="$FILENAME.mp4"
+OUTPUT_DIR=$2
+
+if [ -z "$OUTPUT_DIR" ]; then
+ OUTPUT_DIR="."
+fi
+
+FILENAME=`basename ${INPUT%%.*}`
+OUTPUT="$OUTPUT_DIR/$FILENAME.mp4"
echo "-> Converting $INPUT to $OUTPUT file"
exit 1
fi
+# 720p
+# HandBrakeCLI -d -b 3000 -q 0.8 -w 1280 -i /naspool/videos/HomeVideos/2011/2011-12-01.000.mkv -o 2011-12-01.mp4
+
ffmpeg -i $INPUT \
-b:v 3000k \
-vcodec libx264 \
# SUBROUTINES
sub detect_ext {
- my ($ffmpeg_info, $interlaced) = @_;
+ my ($ffmpeg_info, $progressive) = @_;
my $h264 = 0;
my $h264_high = 0;
# Quicktime/MOV
if($h264 && $pcm_s16le) { return "mov"; }
# MTS
- if($h264_high && $ac3) { if($interlaced) { return "mkv"; } else { return "mp4"; } }
+ if($h264_high && $ac3) { if(!$progressive) { return "mkv"; } else { return "mp4"; } }
# 3GP/MP4
- if($h264 && $aac) { if($interlaced) { return "mkv"; } else { return "mp4"; } }
+ if($h264 && $aac) { if(!$progressive) { return "mkv"; } else { return "mp4"; } }
return "UNKNOWN";
}
if($opt_c =~ /all/) { $all_chapters = 1; }
print "-> Extracting the following chapters from $opt_i: @chapters\n";
-my $interlaced = system('ffmpeg -i "$opt_i" 2>&1 | grep -q "frame rate differs"');
-if($interlaced != 0) {
+my $progressive = system('ffmpeg -i "$opt_i" 2>&1 | grep -q "frame rate differs"');
+if(!$progressive) {
print " Detected interlaced video content\n";
} else {
print " Detected progressive video content\n";
if($all_chapters || &export_chapter(\@chapters,$chapter)) {
$ext = "UNKNOWN";
if(defined $opt_e) { $ext = $opt_e; }
- else { $ext = &detect_ext(\@ffmpeg_info, $interlaced); }
+ else { $ext = &detect_ext(\@ffmpeg_info, $progressive); }
if($ext =~ /UNKNOWN/) { die "-E- Unable to determine the file type/extension to use for the output videos. Specify it with the -e <ext> option.\n"; }
$dstfile = $opt_o . ".c" . sprintf("%03d",$chapter) . "." . $ext;
print "-> Exporting $chapter to $dstfile:\n";
$errno = $errno >> 8;
if($errno > 0) { die "-E- ffmpeg encountered some errors with exit code $errno\n"; }
}
- if($interlaced && $ext =~ /mkv/i) {
+ if(!$progressive && $ext =~ /mkv/i) {
$ext = "mp4";
$srcfile = $dstfile;
$dstfile = $dstfile = $opt_o . ".c" . sprintf("%03d",$chapter) . "." . $ext;
# Find out if content is interlaced or not
$deint_option = "-deinterlace";
- $interlaced = system('ffmpeg -i "$_.MTS" 2>&1 | grep -q "frame rate differs"');
- if($interlaced != 0) {
+ $progressive = system('ffmpeg -i "$_.MTS" 2>&1 | grep -q "frame rate differs"');
+ if(!$progressive) {
print " Detected interlaced video content\n";
} else {
print " Detected progressive video content\n";
my $srcpathname = "/naspool/pictures/New Photos"; # Path to look for videos to move from
my $dstpathname = "/naspool/videos/HomeVideos"; # Path to move the videos to
my $merge_videos_by_day = "/naspool/videos/bin/merge_videos_by_day";
-my $minage = "+30"; # Video file creation dates must not have changed in the last X minutes to process any of the video files
+my $minage = "+0"; # Video file creation dates must not have changed in the last X minutes to process any of the video files
my $owner = "ajp"; # The owner of the files after they are moved
my $group = "pip"; # The group of the files after they are moved
my $mode = "664"; # The mode to set on each file after they are moved
my $merge_opts = "";
if(defined $opt_t) { $merge_opts .= "-t "; }
if(defined $opt_r) { $merge_opts .= "-r "; }
-$errno=system("$merge_videos_by_day -s \"$srcpathname\" $merge_opts");
+$errno=system("$merge_videos_by_day -q -s \"$srcpathname\" $merge_opts");
$errno = $errno >> 8;
if($errno) { die "-E- $merge_videos_by_day encountered some errors with exit code $errno\n"; }
if(! defined $opt_t) {
$tmpdir=`tempfile`;
system("rm $tmpdir");
- system("mv \"$srcdir\" $tmpdir > /dev/null 2>/dev/null");
+ system("cp -R \"$srcdir\" $tmpdir > /dev/null 2>/dev/null");
+ system("rm -rf \"$srcdir\"");
print "-> Moved empty subdir $srcdir to $tmpdir\n" if($opt_v);
}
}
projects / videoscripts / .git / commitdiff