From: Alan J. Pippin Date: Sat, 4 Nov 2017 18:48:59 +0000 (-0600) Subject: Major rewrite of video import scripts to handle videos from any device X-Git-Url: http://git.pippins.net/embedvideo/.git/%7Bupdate_month%7D?a=commitdiff_plain;h=5084e6bde7b0e10b98c08badc2759bff76c74f26;p=videoscripts%2F.git Major rewrite of video import scripts to handle videos from any device Also handles merging videos properly based on their mergeable attributes Also handles rotating videos taken in portrait mode to landscape --- diff --git a/make_mkv b/make_mkv index b947780..f2c2e53 100755 --- a/make_mkv +++ b/make_mkv @@ -13,6 +13,7 @@ use File::Copy; use File::Basename; use Getopt::Std; use File::stat; +use Data::Dumper; use DateTime; use DateTime::Duration; use DateTime::Format::Duration; @@ -38,8 +39,8 @@ if(! -x $avconv) { die "-E- Missing required executable for avconv: $avconv\n"; sub usage { print "usage: $0 -t -o <output.mkv> -i <input,input,...>\n"; print " -t <title> Sets the general title for the output video file\n"; - print " -o <output.mkv> Sets the name of the output mkv file\n"; - print " -i <input,input,...> Sets the name of the input files to make into an mkv file\n"; + print " -o <output> Sets the basename of the output mkv file\n"; + print " -i <input,input,...> Sets the names of the input files to combine into a new mkv file\n"; print " -q Requantize input videos to decrease output video size (requires HandBrakeCLI)\n"; print " -z Recompress input videos to decrease output video size (requires HandBrakeCLI)\n"; print " -s Simulate mode. Don't actually make the video, but tell us what you will do\n"; @@ -48,6 +49,7 @@ sub usage { return 1; } +$SIG{'INT'} = sub {die "-E- Killed by CTRL-C\n"}; #################################################################################################### # Helper Subroutines sub epoch_to_date { @@ -69,47 +71,13 @@ foreach $video (split(/,/, $opt_i)) { $videos{$video} = $mtime_epoch; } -# Create the chapters file for the mkv file. -# Make each video file it's own chapter in the MKV file. -# Name the chapter with the date and time the video clip was taken (modified date). -print "-> Creating $opt_o with title '$opt_t' from the following video files in last modified date order:\n"; -open(CHAPTERS,">$chapter_file") || die "-E- Unable to create chapter file: $chapter_file\n"; -my $chapter_num = 0; -my $chapter_timecode = DateTime::Duration->new(years => 2000, hours => 0, minutes => 0, seconds => 0); -my $timecode_format = DateTime::Format::Duration->new(pattern => '%H:%M:%S.%3N', normalize => 1); -foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { - $chapter_num++; - my $hour = 0; - my $min = 5; - my $sec = 0; - my $mtime_epoch = stat("$video")->mtime; - my $mdate = epoch_to_date($mtime_epoch); - my $duration = `$ffmpeg -i "$video" 2>&1 | grep Duration`; - if($duration =~ /Duration: (\d+):(\d+):(\d+)\.(\d+)/) { - $hour = $1; - $min = $2; - $sec = "$3.$4"; - } - my $timecode = $timecode_format->format_duration($chapter_timecode); - print "$mdate $hour:$min:$sec -> $video \n"; - print CHAPTERS "CHAPTER".sprintf("%02d",$chapter_num)."=".$timecode."\n"; - print CHAPTERS "CHAPTER".sprintf("%02d",$chapter_num)."NAME=".$mdate."\n"; - my $dt = DateTime::Duration->new(years => 2000, hours => $hour, minutes => $min, seconds => $sec); - $chapter_timecode = $chapter_timecode + $dt; -} -close(CHAPTERS); -print "\n-> Creating the following chapter file for this video:\n"; -$chapter_file_contents = `cat $chapter_file`; -print "$chapter_file_contents\n"; - -# Create the mkv video -print "-> Creating the MKV video file '$opt_o'\n"; - -my $cmd = "$mkvmerge --title \"$opt_t\" $output_file_options -o \"$opt_o\""; +#################################################################################################### +# 1) Create the mkv video +print "-> Creating MKV video files with basename '$opt_o'\n"; # Make our input file command line options for all the videos my @video_tmp_files; -my %merge_cmd; +my %merge_videos; foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { # Make a note of the video extension my $video_ext = $video; @@ -152,6 +120,8 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { if($errors == 0) { # Push the name of our intermediate video file onto a list of files to delete on exit push(@video_tmp_files, "$video_mkv"); + # Copy the modification date from the src video to the dst video + system("touch \"$video_mkv\" -r \"$video\""); # Update the name of our video file to equal the name of our new intermediate video file name $video = $video_mkv; } @@ -173,10 +143,20 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { my $video_mp4 = $video; $video_mp4 =~ s/\.[^.]*$//; $video_mp4 .= ".hb.mp4"; # Don't recompress certain file extensions + # Unless the video is not progressive or is rotated, then we do need to recompress it to deinterlace or unrotate it my $no_recompress = 0; - if($video =~ $no_recompress_file_ext) { $no_recompress = 1;} + if($no_recompress_file_ext && $progressive && ($video =~ $no_recompress_file_ext)) { $no_recompress = 1; } + + # Set our rotate option accordingly to rotate videos taken in portrait mode to landscape + my $rotate = `$ffprobe "$video" 2>&1 | grep -e rotate | awk '{print \$3}'`; chomp($rotate); + if($rotate ne "") { + print " Detected rotated input video ($rotate). Rotating video to: $video_mp4\n"; + if($rotate eq "90") { $handbrake_options .= " --rotate=4"; $rotated = "90"; } + if($rotate eq "180") { $handbrake_options .= " --rotate=3"; $rotated = "none"; } + $no_recompress = 0; # We have to recompress this video to unrotate it + } - # Set our requantize factor accordingly + # Set our requantize factor accordingly if(defined $opt_q and ! -f $video_mp4) { print " Re-quantizing input video content to: $video_mp4\n"; $handbrake_options = $handbrake_requantize_options; @@ -195,14 +175,6 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { if($AUDIO_CODEC eq "") { print "-W- Unable to extract audio track encoding from input video file: $video\n"; $handbrake_options .= " -E copy"; } else { $handbrake_options .= " -E copy:$AUDIO_CODEC"; } } - - # Set our rotate option accordingly to rotate videos taken in portrait mode to landscape - my $rotate = `$ffprobe "$video" 2>&1 | grep -e rotate | awk '{print \$3}'`; chomp($rotate); - if($rotate ne "") { - print " Detected rotated input video ($rotate). Rotating video to: $video_mp4\n"; - if($rotate eq "90") { $handbrake_options .= " --rotate=4"; $rotated = "90"; } - if($rotate eq "180") { $handbrake_options .= " --rotate=3"; $rotated = "none"; } - } # Set our de-interlace option accordingly my $deinterlace_option = ""; @@ -221,7 +193,7 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { } } } else { - if(! -f $video_mp4) { + if($no_recompress) { print " Copying input video content to: $video_mp4\n"; system("cp \"$video\" \"$video_mp4\"") if(!$opt_s); } @@ -229,44 +201,128 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { # Push the name of our intermediate video file onto a list of files to delete on exit push(@video_tmp_files, "$video_mp4"); + # Copy the modification date from the src video to the dst video + system("touch \"$video_mp4\" -r \"$video\""); # Update the name of our video file to equal the name of our new intermediate video file name $video = $video_mp4; } + - # Create our mkvmerge input file command line options for this video - if(not defined $merge_cmd{$rotated}) { - $merge_cmd{$rotated} = "$cmd $input_file_options \"$video\""; - } else { - $merge_cmd{$rotated} .= " $input_file_options + \"$video\""; - } + #################################################################################################### + # 2) Group the encoded videos together by their parameters + # We can merge videos that have the same parameters together in a destination video file. + + # Dimensions + my $video_stream_info = `$ffprobe "$video" 2>&1 | grep -e "Stream.*Video"`; chomp($video_stream_info); + my $dimensions = "unknown"; + if($video_stream_info =~ / (\d+x\d+)[,| ]/) { $dimensions = "$1"; } + else { print "-W- ffprobe was unable to find dimensions for video: $video\n"; } + + # Audio Handler + my $audio_stream_info = `$ffprobe "$video" 2>&1 | grep -e "Stream.*Audio"`; chomp($video_stream_info); + my $audio_handler = "unknown"; + if($audio_stream_info =~ / Hz, ([^,]+), /) { $audio_handler = "$1"; } + else { print "-W- ffprobe was unable to find audio handler for video: $video\n"; } + + # Audio Codec + my $audio_codec = "unknown"; + if($audio_stream_info =~ /Audio: (\S+) /) { $audio_codec = "$1"; } + else { print "-W- ffprobe was unable to find audio codec for video: $video\n"; } + + # Now create our parameters string + my $parameters = "$dimensions.$audio_handler.$audio_codec"; + + print " Adding video $video to be merged into output video file: $opt_o.$parameters.mkv\n" if($opt_v); + push @{$merge_videos{"$parameters"}}, $video; +} + +if($opt_v) { + print "\n-> Merge videos hash:\n"; + print Dumper(\%merge_videos); + print "\n"; } -# Execute our command line -foreach my $key (keys %merge_cmd) { - my $merge_cmd = $merge_cmd{$key}; - # if the $key is not none, change the output filename to avoid naming conflicts since it will need to create a separate output file - if($key ne "none") { - $merge_cmd =~ s/-o "(.*?)\.(mkv)"/-o "$1\.$key\.$2"/g; +#################################################################################################### +# 3) Create a chapter file for each destination video file + +# tmp chapter file used by handbrake when creating mkv, but remove the 0 byte file it creates, we'll create it if we need it +my $chapter_file = `tempfile`; chomp($chapter_file); unlink "$chapter_file"; + +foreach my $key (keys %merge_videos) { + # Create the chapters file for each output mkv file. + # Make each video file it's own chapter in the MKV file. + # Name the chapter with the date and time the video clip was taken (modified date). + print "-> Creating $opt_o.$key.mkv with title '$opt_t' from the following video files in last modified date order:\n"; + open(CHAPTERS,">$chapter_file.$key") || die "-E- Unable to create chapter file: $chapter_file.$key\n"; + my $chapter_num = 0; + my $chapter_timecode = DateTime::Duration->new(years => 2000, hours => 0, minutes => 0, seconds => 0); + my $timecode_format = DateTime::Format::Duration->new(pattern => '%H:%M:%S.%3N', normalize => 1); + foreach my $video (@{$merge_videos{$key}}) { + $chapter_num++; + my $hour = 0; + my $min = 5; + my $sec = 0; + my $mtime_epoch = stat("$video")->mtime; + my $mdate = epoch_to_date($mtime_epoch); + my $duration = `$ffmpeg -i "$video" 2>&1 | grep Duration`; + if($duration =~ /Duration: (\d+):(\d+):(\d+)\.(\d+)/) { + $hour = $1; + $min = $2; + $sec = "$3.$4"; + } + my $timecode = $timecode_format->format_duration($chapter_timecode); + print "$mdate $hour:$min:$sec -> $video \n"; + print CHAPTERS "CHAPTER".sprintf("%02d",$chapter_num)."=".$timecode."\n"; + print CHAPTERS "CHAPTER".sprintf("%02d",$chapter_num)."NAME=".$mdate."\n"; + my $dt = DateTime::Duration->new(years => 2000, hours => $hour, minutes => $min, seconds => $sec); + $chapter_timecode = $chapter_timecode + $dt; + } + close(CHAPTERS); + print "\n-> Creating the following chapter file $chapter_file.$key for this video:\n"; + $chapter_file_contents = `cat $chapter_file.$key`; + print "$chapter_file_contents\n"; +} + +#################################################################################################### +# 4) Run mkvmerge to merge all of the videos together of the same parameters +foreach my $key (keys %merge_videos) { + my $merge_cmd = "$mkvmerge --title \"$opt_t\" $output_file_options --chapters $chapter_file.$key -o \"$opt_o.$key.mkv\""; + my $i = 0; + foreach my $video (@{$merge_videos{$key}}) { + if($i == 0) { + $merge_cmd .= " $input_file_options \"$video\""; + } else { + $merge_cmd .= " $input_file_options + \"$video\""; + } + $i++; } print "\n$merge_cmd\n"; if(! defined $opt_s) { + if(-f "$opt_o.$key.mkv") { die "-E- Output video file $opt_o already exists. This is unexpected for key $key!"; } my $errno = system("$merge_cmd"); $errno = $errno >> 8; if($errno > 1) { - unlink "$opt_o"; + unlink "$opt_o.$key.mkv"; die "-E- mkvmerge encountered some errors with exit code $errno\n"; } } } -# Remove the temporary file used for ffmpeg and handbrake output -if(-e "$tmpfile") { unlink "$tmpfile"; } -# Remove the temporary file used for the chapter generation -if(-e "$chapter_file") { unlink "$chapter_file"; } +#################################################################################################### +# 5) Remove all temporary files +if(!$opt_s) { + # Remove the temporary file used for ffmpeg and handbrake output + if(-e "$tmpfile") { unlink "$tmpfile"; } + + # Remove the temporary file used for the chapter generation + foreach my $key (keys %merge_videos) { + if(-e "$chapter_file.$key") { unlink "$chapter_file.$key"; } + } -# Remove any temporary video files created during the merge process -foreach my $video (@video_tmp_files) { - if(-e "$video") { unlink "$video"; } + # Remove any temporary video files created during the merge process + foreach my $video (@video_tmp_files) { + if(-e "$video") { unlink "$video"; } + } } #################################################################################################### diff --git a/merge_videos_by_day b/merge_videos_by_day index 1a153b0..cda301d 100755 --- a/merge_videos_by_day +++ b/merge_videos_by_day @@ -34,7 +34,9 @@ sub usage { return 1; } if(defined $opt_h) { usage(); exit 1; } +$SIG{'INT'} = sub {die "-E- Killed by CTRL-C\n"}; +#################################################################################################### # Sanity checks if((defined $opt_q || defined $opt_z) && !$use_compute_host && ! -x "$handbrake") { die "-E- Unable to find required program: handbrake\n"; } if(! -d $srcpathname) { &usage; print "-E- Can't find srcpath: $srcpathname\n"; exit 1; } @@ -136,137 +138,82 @@ foreach $file (sort `$find_cmd`) { $dstdir = $srcdir; $dstfile = $dstdir . "/" . $year . "-" . $monthnum . "-" . $day; - # Check for duplicate filenames at the destination - $newfile = $dstfile . "." . $video_suffix; - if(-e "$newfile.$ext") { - foreach $i ($video_suffix+1 .. '999') { - $newfile = $dstfile . "." . sprintf("%03d",$i); - if(! -e "$newfile.$ext") { last; } - } - $dstfile = $newfile; - } - - # Set the name of our unique destination file - $dstfile = "$newfile.$ext"; - - # You can only merge videos into a single destination that have the same extension/type - push(@{$videos{"$srcext"}{"$dstfile"}}, "\"$srcdir/$srcfile\""); -} - -# Check for duplicate filenames in the dstfiles being created for other exts -my $last_dstnum = 0; -my $changed_dst = 0; -foreach $ext (sort keys %videos) { - #print "checking: $ext\n"; - foreach $video (sort keys %{$videos{$ext}}) { - #print "checking: $ext $video\n"; - # Make sure this video name is not in use as a destination for any other ext - foreach $checkext (sort keys %videos) { - #print "checking: $ext $video $checkext\n"; - if($checkext eq $ext) { next; } - foreach $checkvideo (sort keys %{$videos{$checkext}}) { - #print "checking: $ext $video $checkext $checkvideo\n"; - if("$video" eq "$checkvideo") { - if($video =~ /(.*?)\.(\d+)\.(\w+)$/) { - $dstfile = $1; - $dstnum = $2; - $dstext = $3; - } - foreach $i ($last_dstnum .. '999') { - $last_dstnum = $i + 1; - $newfile = $dstfile . "." . sprintf("%03d",$i); - if("$video" ne "$newfile.$dstext") { last; } - } - $videos{$ext}{"$newfile.$dstext"} = $videos{$ext}{$video}; - #print "for ext $ext changed destination to: $newfile.$dstext: $videos{$ext}{$video};\n"; - delete $videos{$ext}{$video}; - $changed_dst = 1; - last; - } - } - if($changed_dst) { last; } - } - if($changed_dst) { last; } - } - $changed_dst = 0; + # Make a note of this video and its merged destination + push(@{$videos{"$dstfile"}}, "\"$srcdir/$srcfile\""); } # Tell the user which videos we are going to merge -foreach $ext (sort keys %videos) { - foreach $video (sort keys %{$videos{$ext}}) { - foreach $srcfile (@{$videos{$ext}{$video}}) { - print " merging $ext $srcfile into \"$video\"\n"; - } +foreach $dstfile (sort keys %{$videos}) { + foreach $srcfile (@{$videos{$dstfile}}) { + print " merging $srcfile into \"$dstfile\"\n"; } } # Now actually do the merging print "\n"; -foreach $ext (sort keys %videos) { - foreach $video (sort keys %{$videos{$ext}}) { +foreach $dstfile (sort keys %videos) { - my $videos = join(',', @{$videos{$ext}{$video}}); - - if($video =~ /(\d+)-(\d+)-(\d+)/) { - $year = $1; - $month = $2; - $day = sprintf("%02d",$3); - } - - my $pwd = `pwd`; chomp($pwd); - my $cmd = ""; - if($use_compute_host) { $cmd .= "ssh $compute_host 'cd \"$pwd\";"; } - $cmd .= "$make_mkv -t \"$video_title_prefix $year-$month-$day\" -o \"$video\" -i $videos"; - if($requantize_input_video) { $cmd .= ' -q'; } - if($recompress_input_video) { $cmd .= ' -z'; } - if($opt_v) { $cmd .= ' -v'; } - if($use_compute_host) { $cmd .= "'"; } - if(defined $opt_t) { - # Print what will be done, but don't actually do it - print "\n-> Creating \"$video\"\n"; - print "$cmd\n"; - if(!defined $opt_k) { - foreach $video (@{$videos{$ext}{$video}}) { - if(($save_originals) && ($video =~ /\.$originals_file_ext/)) { - print "-> Saving the original video $video\n"; - if(index($video, basename(dirname($video))) == -1) { - print("mv $video \"$origpathname/".basename(dirname($video))."_".basename($video)."\n"); - } else { - print("mv $video \"$origpathname/".basename($video)."\n"); - } + my $videos = join(',', @{$videos{$dstfile}}); + + if($dstfile =~ /(\d+)-(\d+)-(\d+)/) { + $year = $1; + $month = $2; + $day = sprintf("%02d",$3); + } + + my $pwd = `pwd`; chomp($pwd); + my $cmd = ""; + if($use_compute_host) { $cmd .= "ssh $compute_host 'cd \"$pwd\";"; } + $cmd .= "$make_mkv -t \"$video_title_prefix $year-$month-$day\" -o \"$dstfile\" -i $videos"; + if($requantize_input_video) { $cmd .= ' -q'; } + if($recompress_input_video) { $cmd .= ' -z'; } + if($opt_v) { $cmd .= ' -v'; } + if($use_compute_host) { $cmd .= "'"; } + if(defined $opt_t) { + # Print what will be done, but don't actually do it + print "\n-> Creating \"$dstfile\"\n"; + print "$cmd\n"; + if(!defined $opt_k) { + foreach $video (@{$videos{$dstfile}}) { + if(($save_originals) && ($video =~ /\.$originals_file_ext/)) { + print "-> Saving the original video $video\n"; + if(index($video, basename(dirname($video))) == -1) { + print("mv $video \"$origpathname/".basename(dirname($video))."_".basename($video)."\n"); } else { - print "-> Removing the original video $video\n"; - print("/bin/bash -c '[[ -e $video ]] && rm -f $video'\n"); + print("mv $video \"$origpathname/".basename($video)."\n"); } + } else { + print "-> Removing the original video $video\n"; + print("/bin/bash -c '[[ -e $video ]] && rm -f $video'\n"); } } - } else { - # Create the merged video - print "$ext: $cmd" if($opt_v); - my $errno = system("$cmd"); - $errno = $errno >> 8; - if($errno) { die "-E- make_mkv encountered some errors with exit code $errno\n"; } - system("ls -l \"$srcpathname/\" > /dev/null"); # Make sure the video file is there - # Fix the permissions - if(-f "$video") { - system("chown $owner \"$video\""); - system("chgrp $group \"$video\""); - system("chmod $mode \"$video\""); - } - # Remove the individual video files - if(!defined $opt_k) { - foreach $srcvideo (@{$videos{$ext}{$video}}) { - if(($save_originals) && ($srcvideo =~ /\.$originals_file_ext/)) { - print "-> Saving the original video $srcvideo to $origpathname\n"; - if(index($srcvideo, basename(dirname($srcvideo))) == -1) { - system("mv $srcvideo \"$origpathname/".basename(dirname($srcvideo))."_".basename($srcvideo)); - } else { - system("mv $srcvideo \"$origpathname/".basename($srcvideo)); - } + } + } else { + # Create the merged video + print "$cmd" if($opt_v); + my $errno = system("$cmd"); + $errno = $errno >> 8; + if($errno) { die "-E- make_mkv encountered some errors with exit code $errno\n"; } + system("ls -l \"$srcpathname/\" > /dev/null"); # Make sure the video file is there + + # Fix the permissions + system("chown $owner \"$dstfile\"*"); + system("chgrp $group \"$dstfile\"*"); + system("chmod $mode \"$dstfile\"*"); + + # Remove the individual video files + if(!defined $opt_k) { + foreach $srcvideo (@{$videos{$dstfile}}) { + if(($save_originals) && ($srcvideo =~ /\.$originals_file_ext/)) { + print "-> Saving the original video $srcvideo to $origpathname\n"; + if(index($srcvideo, basename(dirname($srcvideo))) == -1) { + system("mv $srcvideo \"$origpathname/".basename(dirname($srcvideo))."_".basename($srcvideo)); } else { - print "-> Removing the original video $srcvideo\n"; - system("/bin/bash -c '[[ -e $srcvideo ]] && rm -f $srcvideo'"); + system("mv $srcvideo \"$origpathname/".basename($srcvideo)); } + } else { + print "-> Removing the original video $srcvideo\n"; + system("/bin/bash -c '[[ -e $srcvideo ]] && rm -f $srcvideo'"); } } } diff --git a/organize_videos b/organize_videos index 34fd2e6..ea92012 100755 --- a/organize_videos +++ b/organize_videos @@ -67,6 +67,7 @@ sub usage { print " -d <dir> directory to recreate the playlists in. Only needed if -p option is given\n"; return 1; } +$SIG{'INT'} = sub {die "-E- Killed by CTRL-C\n"}; sub is_folder_empty { my $dirname = shift; @@ -233,7 +234,6 @@ foreach $file (`$find_mkv`) { } $dstfile = "$newfile.$ext"; - if(defined $opt_t) { print "-> Moving \"$srcdir/$srcfile\" to \"$dstfile\"\n"; } else { diff --git a/organize_videos.conf b/organize_videos.conf index 9216f4b..54a57a4 100644 --- a/organize_videos.conf +++ b/organize_videos.conf @@ -95,14 +95,11 @@ $timezone = `cat /etc/timezone`; chomp($timezone); $handbrake_requantize_options='--strict-anamorphic --crop 0:0:0:0 -E ac3'; $handbrake_recompress_options='--strict-anamorphic --crop 0:0:0:0 --denoise="weak" -e x264 -q 18 -x b-adapt=2:rc-lookahead=50 -v 2 -a 1 -6 dpl2 --preset="High Profile"'; -# tmp chapter file used by handbrake when creating mkv, but remove the 0 byte file it creates, we'll create it if we need it -$chapter_file = `tempfile`; chomp($chapter_file); unlink "$chapter_file"; - # handbrake input file options $input_file_options = "-S"; # handbrake output file options -$output_file_options = "--chapters $chapter_file --compression -1:none"; +$output_file_options = "--compression -1:none"; # handbrake quantization levels $interlaced_requantize_quality=0.85;