From: Alan J. Pippin Date: Fri, 13 Oct 2017 01:46:51 +0000 (-0600) Subject: Fixed up scripts to support importing videos from phones X-Git-Url: http://git.pippins.net/embedvideo/.git/static/images/%7Bdone_action%7D?a=commitdiff_plain;h=bfa24cc8c0228028cae669b2fa0adddef10470ae;p=videoscripts%2F.git Fixed up scripts to support importing videos from phones --- diff --git a/copy_videos_from_watchdir b/copy_videos_from_watchdir new file mode 100755 index 0000000..e64a505 --- /dev/null +++ b/copy_videos_from_watchdir @@ -0,0 +1,87 @@ +#!/usr/bin/perl +# Author: Alan J. Pippin +# Description: Find videos from a watched dir and copy them to the video processing destination directory + +use File::Copy; +use File::Basename; +use Getopt::Std; +use File::stat; +use Time::localtime; +use File::Pid; + +#################################################################################################### +# Configuration parameters +$mydir = `cd \$(dirname $0) 2>/dev/null; pwd`; chomp($mydir); unshift @INC,("$mydir"); +# Default configuration values +require "organize_videos.conf"; +# Override defaults with local customizations +if( -f "$mydir/organize_videos.conf.local") { require "organize_videos.conf.local"; } + +#################################################################################################### + +# Sanity check +if(! -d $srcpathname) { print "-E- Can't find srcpath: $srcpathname\n"; exit 1; } +if(! -d $watchpathname) { print "-E- Can't find watchpath: $watchpathname\n"; exit 1; } + +getopts("hvt"); + +sub usage { + print "usage: $0 [-v] [-t]\n"; + print " -v verbose; print file being copied (to).\n"; + print " -t test; print what will happen, but don't do anything\n"; + return 1; +} + +# Sanity checks / Option processing +if(defined $opt_h) { usage(); exit 1; } + +# Our srcpathname is actually our dstpathname +$dstpathname = $srcpathname; +$srcpathname = $watchpathname; + +# Only proceed if there are video files to organize +$video_files_found=`find \"$srcpathname/\" $movie_file_ext -o -iregex \".*\.mkv\"`; +if(!$video_files_found) { exit 0; } + +# Only one instance of this script running at a time +my $pidfile = File::Pid->new({file => "$pid_file", pid => "$$"}); +exit if $pidfile->running(); +$pidfile->write(); + +# Print the date +system("date"); + +# Merge videos prior to copying them over to the destination path +my $errno = 0; + +# Copy the videos over to the destination path +chdir "$srcpathname"; +print "$video_files_found\n" if($opt_v); +foreach $file (`find \"$srcpathname/\" $movie_file_ext -o -iregex \".*\.mkv\"`) { + + chomp($file); + $srcdir = dirname($file); + $file = basename($file); + $srcfile = $file; + $dstfile = "$dstpathname/$file"; + + print "-> Copying \"$srcdir/$srcfile\" to \"$dstfile\"\n"; + + if(!defined $opt_t) { + # Make sure the dstfile doesn't exist, if it does, don't do the copy + if(! -f "$dstfile") { + $errno=system("cp -p \"$srcdir/$srcfile\" \"$dstfile\" 2>/dev/null"); + if($errno) { print "-E- Error copying srcfile to dstfile: $srcdir/$srcfile -> $dstfile\n"; next; } + } else { + print "-> Skipping \"$srcdir/$srcfile\". Destfile \"$dstfile\" already exists.\n"; + } + # Fix the permissions + system("chown $owner \"$dstfile\""); + system("chgrp $group \"$dstfile\""); + system("chmod $mode \"$dstfile\""); + } +} + +$pidfile->remove(); + +print "\n\n"; diff --git a/make_mkv b/make_mkv index 805a7f5..b947780 100755 --- a/make_mkv +++ b/make_mkv @@ -108,8 +108,8 @@ print "-> Creating the MKV video file '$opt_o'\n"; my $cmd = "$mkvmerge --title \"$opt_t\" $output_file_options -o \"$opt_o\""; # Make our input file command line options for all the videos -my $video_count = 0; my @video_tmp_files; +my %merge_cmd; foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { # Make a note of the video extension my $video_ext = $video; @@ -135,21 +135,26 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { # We don't need to do this anymore since it is not an issue with the new mkvmerge if($video_ext !~ /mkv/i) { $video_mkv =~ s/\.[^.]*$//; $video_mkv .= ".ffmpeg.mkv"; + if(-f $video_mkv) { next; } print " Re-muxing the interlaced video content as an mkv file: $video_mkv\n"; my $make_mkv_cmd = "$avconv -y -i \"$video\" -scodec copy -acodec copy -vcodec copy -f matroska \"$video_mkv\" >> \"$tmpfile\" 2>&1"; + my $errors = 0; if($opt_v) { print " $make_mkv_cmd\n"; } if(! defined $opt_s) { my $errno = system("$make_mkv_cmd"); $errno = $errno >> 8; if($errno > 1) { unlink "$video_mkv"; - die "-E- ffmpeg encountered some errors with exit code $errno\n"; + print "-W- avconv encountered some errors with exit code $errno\n"; + $errors = 1; } } - # Push the name of our intermediate video file onto a list of files to delete on exit - push(@video_tmp_files, "$video_mkv"); - # Update the name of our video file to equal the name of our new intermediate video file name - $video = $video_mkv; + if($errors == 0) { + # Push the name of our intermediate video file onto a list of files to delete on exit + push(@video_tmp_files, "$video_mkv"); + # Update the name of our video file to equal the name of our new intermediate video file name + $video = $video_mkv; + } } } else { print " Detected progressive video content: $video\n"; @@ -159,6 +164,7 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { # This also gives us a chance to deinterlace the video as well # Only re-quantize for .MTS videos # We can re-compress any input video + my $rotated = "none"; if(((defined $opt_q) && ($video_ext =~ /mts/i)) || (defined $opt_z)) { my $handbrake_options = ""; @@ -166,8 +172,12 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { # Set our output video filename my $video_mp4 = $video; $video_mp4 =~ s/\.[^.]*$//; $video_mp4 .= ".hb.mp4"; + # Don't recompress certain file extensions + my $no_recompress = 0; + if($video =~ $no_recompress_file_ext) { $no_recompress = 1;} + # Set our requantize factor accordingly - if(defined $opt_q) { + if(defined $opt_q and ! -f $video_mp4) { print " Re-quantizing input video content to: $video_mp4\n"; $handbrake_options = $handbrake_requantize_options; if(!$progressive) { $handbrake_options = "-q $interlaced_requantize_quality"; } @@ -175,14 +185,23 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { }; # Set our recompress options accordingly - if(defined $opt_z) { + if(defined $opt_z and ! -f $video_mp4 and ! $no_recompress) { print " Re-compressing input video content to: $video_mp4\n"; $handbrake_options = $handbrake_recompress_options; # We want our audio to be passed-through by default, so detect how the audio of the input is encoded, and tell handbrake to make the output match - if($opt_v) { print " $ffmpeg -i \"$video\" 2>&1 | grep \"Audio\" | sed -r -e 's/.*?Audio: (\\S+).*?/\\1/'\n"; } - $AUDIO_CODEC=`$ffmpeg -i "$video" 2>&1 | grep "Audio" | sed -r -e 's/.*?Audio: (\\S+).*?/\\1/'`; chomp($AUDIO_CODEC); - if($AUDIO_CODEC eq "") { die "-E- Unable to extract audio track encoding from input video file: $video\n"; } - $handbrake_options .= " -E copy:$AUDIO_CODEC"; + if($opt_v) { print " $ffmpeg -i \"$video\" 2>&1 | grep \"Audio\" | sed -r -e 's/.*?Audio: (\\S+).*?/\\1/' | tail -n 1\n"; } + $AUDIO_CODEC=`$ffmpeg -i "$video" 2>&1 | grep "Audio" | sed -r -e 's/.*?Audio: (\\S+).*?/\\1/' | tail -n 1`; chomp($AUDIO_CODEC); + $AUDIO_CODEC =~ s/,$//g; + if($AUDIO_CODEC eq "") { print "-W- Unable to extract audio track encoding from input video file: $video\n"; $handbrake_options .= " -E copy"; } + else { $handbrake_options .= " -E copy:$AUDIO_CODEC"; } + } + + # Set our rotate option accordingly to rotate videos taken in portrait mode to landscape + my $rotate = `$ffprobe "$video" 2>&1 | grep -e rotate | awk '{print \$3}'`; chomp($rotate); + if($rotate ne "") { + print " Detected rotated input video ($rotate). Rotating video to: $video_mp4\n"; + if($rotate eq "90") { $handbrake_options .= " --rotate=4"; $rotated = "90"; } + if($rotate eq "180") { $handbrake_options .= " --rotate=3"; $rotated = "none"; } } # Set our de-interlace option accordingly @@ -190,14 +209,21 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { if(!$progressive) { $deinterlace_option = "-d"; } # Use HandBrake to requantize/recompress/deinterlace the input video - my $handbrake_cmd = "$handbrake $deinterlace_option $handbrake_options -i \"$video\" -o \"$video_mp4\" >> \"$tmpfile\" 2>&1"; - if($opt_v) { print " $handbrake_cmd\n"; } - if(! defined $opt_s) { - my $errno = system("$handbrake_cmd"); - $errno = $errno >> 8; - if($errno > 1) { - unlink "$video_mp4"; - die "-E- handbrake encountered some errors with exit code $errno\n"; + if(! $no_recompress and ! -f $video_mp4) { + my $handbrake_cmd = "$handbrake $deinterlace_option $handbrake_options -i \"$video\" -o \"$video_mp4\" >> \"$tmpfile\" 2>&1"; + if($opt_v) { print " $handbrake_cmd\n"; } + if(! defined $opt_s and ! -f $video_mp4) { + my $errno = system("$handbrake_cmd"); + $errno = $errno >> 8; + if($errno > 1) { + unlink "$video_mp4"; + die "-E- handbrake encountered some errors with exit code $errno\n"; + } + } + } else { + if(! -f $video_mp4) { + print " Copying input video content to: $video_mp4\n"; + system("cp \"$video\" \"$video_mp4\"") if(!$opt_s); } } @@ -207,23 +233,29 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { $video = $video_mp4; } - # Create our input file command line options for this video - if($video_count == 0) { - $cmd .= " $input_file_options \"$video\""; + # Create our mkvmerge input file command line options for this video + if(not defined $merge_cmd{$rotated}) { + $merge_cmd{$rotated} = "$cmd $input_file_options \"$video\""; } else { - $cmd .= " $input_file_options + \"$video\""; + $merge_cmd{$rotated} .= " $input_file_options + \"$video\""; } - $video_count++; } # Execute our command line -print "\n$cmd\n"; -if(! defined $opt_s) { - my $errno = system("$cmd"); - $errno = $errno >> 8; - if($errno > 1) { - unlink "$opt_o"; - die "-E- mkvmerge encountered some errors with exit code $errno\n"; +foreach my $key (keys %merge_cmd) { + my $merge_cmd = $merge_cmd{$key}; + # if the $key is not none, change the output filename to avoid naming conflicts since it will need to create a separate output file + if($key ne "none") { + $merge_cmd =~ s/-o "(.*?)\.(mkv)"/-o "$1\.$key\.$2"/g; + } + print "\n$merge_cmd\n"; + if(! defined $opt_s) { + my $errno = system("$merge_cmd"); + $errno = $errno >> 8; + if($errno > 1) { + unlink "$opt_o"; + die "-E- mkvmerge encountered some errors with exit code $errno\n"; + } } } diff --git a/merge_videos_by_day b/merge_videos_by_day index 7179746..1a153b0 100755 --- a/merge_videos_by_day +++ b/merge_videos_by_day @@ -84,11 +84,15 @@ foreach $file (sort `$find_cmd`) { $srcext = ""; if($srcfile =~ /\.(\w+)$/) { $srcext = $1; } $ext = "mkv"; - - print "Found movie: srcdir: $srcdir srcfile: $srcfile srcext: $srcext dstext: $ext\n" if($opt_v); # Throw out files not in the current srcpath if((! -f "$srcfile") && (! -f "$srcdir/$srcfile")) { next; } + + # Throw out encoded files left over from a previous run + print "srcfile: $srcfile\n"; + if($srcfile =~ /.hb.mp4/) { next; } + + print "Found movie: srcdir: $srcdir srcfile: $srcfile srcext: $srcext dstext: $ext\n" if($opt_v); # Make a note of the month, year, and day this video was taken (from the modification time of the file) $date_taken = ctime(stat("$srcdir/$srcfile")->mtime); @@ -122,7 +126,7 @@ foreach $file (sort `$find_cmd`) { $month = $1; $day = sprintf("%02d",$2); $monthnum = $monthname2month{$month}; - $monthname = lc($month2monthname{$month}); + $monthname = lc($month); } else { print "-E- Unable to parse year and month from this file: $srcdir/$srcfile\n"; next; @@ -150,35 +154,48 @@ foreach $file (sort `$find_cmd`) { } # Check for duplicate filenames in the dstfiles being created for other exts +my $last_dstnum = 0; +my $changed_dst = 0; foreach $ext (sort keys %videos) { - foreach $video (sort keys %{$videos{$ext}}) { + #print "checking: $ext\n"; + foreach $video (sort keys %{$videos{$ext}}) { + #print "checking: $ext $video\n"; # Make sure this video name is not in use as a destination for any other ext foreach $checkext (sort keys %videos) { + #print "checking: $ext $video $checkext\n"; if($checkext eq $ext) { next; } foreach $checkvideo (sort keys %{$videos{$checkext}}) { + #print "checking: $ext $video $checkext $checkvideo\n"; if("$video" eq "$checkvideo") { if($video =~ /(.*?)\.(\d+)\.(\w+)$/) { $dstfile = $1; $dstnum = $2; $dstext = $3; } - foreach $i ($dstnum .. '999') { + foreach $i ($last_dstnum .. '999') { + $last_dstnum = $i + 1; $newfile = $dstfile . "." . sprintf("%03d",$i); if("$video" ne "$newfile.$dstext") { last; } } $videos{$ext}{"$newfile.$dstext"} = $videos{$ext}{$video}; + #print "for ext $ext changed destination to: $newfile.$dstext: $videos{$ext}{$video};\n"; delete $videos{$ext}{$video}; + $changed_dst = 1; + last; } } + if($changed_dst) { last; } } + if($changed_dst) { last; } } + $changed_dst = 0; } # Tell the user which videos we are going to merge foreach $ext (sort keys %videos) { foreach $video (sort keys %{$videos{$ext}}) { foreach $srcfile (@{$videos{$ext}{$video}}) { - print " merging $srcfile into \"$video\"\n"; + print " merging $ext $srcfile into \"$video\"\n"; } } } @@ -202,6 +219,7 @@ foreach $ext (sort keys %videos) { $cmd .= "$make_mkv -t \"$video_title_prefix $year-$month-$day\" -o \"$video\" -i $videos"; if($requantize_input_video) { $cmd .= ' -q'; } if($recompress_input_video) { $cmd .= ' -z'; } + if($opt_v) { $cmd .= ' -v'; } if($use_compute_host) { $cmd .= "'"; } if(defined $opt_t) { # Print what will be done, but don't actually do it @@ -224,14 +242,17 @@ foreach $ext (sort keys %videos) { } } else { # Create the merged video + print "$ext: $cmd" if($opt_v); my $errno = system("$cmd"); $errno = $errno >> 8; if($errno) { die "-E- make_mkv encountered some errors with exit code $errno\n"; } system("ls -l \"$srcpathname/\" > /dev/null"); # Make sure the video file is there # Fix the permissions - system("chown $owner \"$video\""); - system("chgrp $group \"$video\""); - system("chmod $mode \"$video\""); + if(-f "$video") { + system("chown $owner \"$video\""); + system("chgrp $group \"$video\""); + system("chmod $mode \"$video\""); + } # Remove the individual video files if(!defined $opt_k) { foreach $srcvideo (@{$videos{$ext}{$video}}) { diff --git a/move_videos_from_watchdir b/move_videos_from_watchdir new file mode 100755 index 0000000..9c1a242 --- /dev/null +++ b/move_videos_from_watchdir @@ -0,0 +1,98 @@ +#!/usr/bin/perl +# Author: Alan J. Pippin +# Description: Find videos from a watched dir and move them to the video processing destination directory + +use File::Copy; +use File::Basename; +use Getopt::Std; +use File::stat; +use Time::localtime; +use File::Pid; + +#################################################################################################### +# Configuration parameters +$mydir = `cd \$(dirname $0) 2>/dev/null; pwd`; chomp($mydir); unshift @INC,("$mydir"); +# Default configuration values +require "organize_videos.conf"; +# Override defaults with local customizations +if( -f "$mydir/organize_videos.conf.local") { require "organize_videos.conf.local"; } + +#################################################################################################### + +# Sanity check +if(! -d $srcpathname) { print "-E- Can't find srcpath: $srcpathname\n"; exit 1; } +if(! -d $watchpathname) { print "-E- Can't find watchpath: $watchpathname\n"; exit 1; } + +getopts("hfvt"); + +sub usage { + print "usage: $0 [-v] [-t]\n"; + print " -v verbose; print file being moved (to).\n"; + print " -f force it to run by ignoring the \$minage setting in organize_videos.conf\n"; + print " -t test; print what will happen, but don't do anything\n"; + return 1; +} + +# Sanity checks / Option processing +if(defined $opt_h) { usage(); exit 1; } + +# Our srcpathname is actually our dstpathname +$find_changed_cmd =~ s/$srcpathname/$watchpathname/g; +$find_cmd_with_mkv =~ s/$srcpathname/$watchpathname/g; +$dstpathname = $srcpathname; +$srcpathname = $watchpathname; + +# Only proceed if no files have changed in the past $cmin minutes +$changed_files_found=`$find_changed_cmd`; +if(!$opt_f && $changed_files_found) { exit 0; } + +# Only proceed if there are video files to organize +$video_files_found=`$find_cmd_with_mkv`; +if(!$video_files_found) { exit 0; } + +# Only one instance of this script running at a time +my $pidfile = File::Pid->new({file => "$pid_file", pid => "$$"}); +print "pid_file: $pid_file\n" if($opt_v); +exit if $pidfile->running(); +$pidfile->write(); + +# Print the date +system("date"); + +# Merge videos prior to copying them over to the destination path +my $errno = 0; + +# Move the videos over to the destination path +chdir "$srcpathname"; +print "$video_files_found\n" if($opt_v); +foreach $file (`$find_cmd_with_mkv`) { + + chomp($file); + $srcdir = dirname($file); + $file = basename($file); + $srcfile = $file; + $dstfile = "$dstpathname/$file"; + + # only move files that have been organized + if($srcfile !~ /\d\d\d\d-\d\d-\d\d/) { next; } + + print "-> Moving \"$srcdir/$srcfile\" to \"$dstfile\"\n"; + + if(!defined $opt_t) { + # Make sure the dstfile doesn't exist, if it does, don't do the move + if(! -f "$dstfile") { + $errno=system("mv \"$srcdir/$srcfile\" \"$dstfile\" 2>/dev/null"); + if($errno) { print "-E- Error moving srcfile to dstfile: $srcdir/$srcfile -> $dstfile\n"; next; } + } else { + print "-> Skipping \"$srcdir/$srcfile\". Destfile \"$dstfile\" already exists.\n"; + } + # Fix the permissions + system("chown $owner \"$dstfile\""); + system("chgrp $group \"$dstfile\""); + system("chmod $mode \"$dstfile\""); + } +} + +$pidfile->remove(); + +print "\n\n"; diff --git a/organize_videos b/organize_videos index 647e08d..34fd2e6 100755 --- a/organize_videos +++ b/organize_videos @@ -144,6 +144,7 @@ my $errno = 0; my $merge_opts = ""; if(defined $opt_t) { $merge_opts .= "-t "; } if(defined $opt_k) { $merge_opts .= "-k "; } +if(defined $opt_v) { $merge_opts .= "-v "; } if($requantize_input_video) { $merge_opts .= ' -q'; } if($recompress_input_video) { $merge_opts .= ' -z'; } $errno=system("$merge_videos_by_day -s \"$srcpathname\" $merge_opts"); @@ -160,8 +161,8 @@ if(defined $opt_n) { # Copy the videos over to the destination path my %dstdirs; chdir "$srcpathname"; -print "$find_cmd_with_mkv\n" if($opt_v); -foreach $file (`$find_cmd_with_mkv`) { +print "$find_mkv\n" if($opt_v); +foreach $file (`$find_mkv`) { chomp($file); $srcdir = dirname($file); @@ -169,10 +170,14 @@ foreach $file (`$find_cmd_with_mkv`) { $srcfile = $file; $ext = $file; $ext =~ s/.*\.(\S+)$/$1/; $ext = lc($ext); - print "Found movie: srcdir: $srcdir srcfile: $srcfile ext: $ext\n" if($opt_v); - # Throw out files not in the current srcpath if((! -f "$srcfile") && (! -f "$srcdir/$srcfile")) { next; } + + # Throw out encoded files left over from a previous run + print "srcfile: $srcfile\n"; + if($srcfile =~ /.hb.mp4/) { next; } + + print "Found movie: srcdir: $srcdir srcfile: $srcfile ext: $ext\n" if($opt_v); # Make a note of the month, year, and day this video was taken (from the modification time of the file) $date_taken = ctime(stat("$srcdir/$srcfile")->mtime); @@ -206,7 +211,7 @@ foreach $file (`$find_cmd_with_mkv`) { $month = $1; $day = sprintf("%02d",$2); $monthnum = $monthname2month{$month}; - $monthname = lc($month2monthname{$month}); + $monthname = lc($month); } else { print "-E- Unable to parse year and month from this file: $srcdir/$srcfile\n"; next; diff --git a/organize_videos.conf b/organize_videos.conf index 702a752..9216f4b 100644 --- a/organize_videos.conf +++ b/organize_videos.conf @@ -19,9 +19,12 @@ $dstpathname = "/naspool/videos/HomeVideos"; # Path to move the originals to $origpathname = "/naspool/dropbox/Originals"; -$originals_file_ext = qr/mov/i; +$originals_file_ext = qr/(mov|mp4)/i; $save_originals = 1; +# Path to a dir to watch for videos to move to $srcpathname to be organized +$watchpathname = "/naspool/pictures/mylio"; + # Path to merge_videos_by_day script $merge_videos_by_day = "/naspool/videos/bin/merge_videos_by_day"; @@ -36,6 +39,7 @@ $mkvmerge = '/usr/bin/mkvmerge'; # ffmpeg path/command name $ffmpeg = '/usr/bin/ffmpeg'; +$ffprobe = '/usr/bin/ffprobe'; $avconv = '/usr/bin/avconv'; # handbrake path/command name @@ -64,7 +68,10 @@ $video_suffix = "000"; # Which movie file extensions should be considered for merging # NOTE: Script does not support merging multiple mkv files into a single mkv file -$movie_file_ext = "-iregex \".*\.mov\" -o -iregex \".*\.3gp\" -o -iregex \".*\.mp4\" -o -iregex \".*\.mts\" -o -iregex \".*\.m4v\""; +$movie_file_ext = "-iregex \".*\.mov\" -o -iregex \".*\.3gp\" -o -iregex \".*\.avi\" -o -iregex \".*\.mp4\" -o -iregex \".*\.mpg\" -o -iregex \".*\.mts\" -o -iregex \".*\.m4v\""; + +# Which movie file extensions should not be recompressed for merging +$no_recompress_file_ext = qr/\.(mp4)$/; # Video file creation dates must not have changed in the last X minutes to process any of the video files # This is done to ensure that all videos from a given upload from a camera have completed prior to looking for videos to merge @@ -76,6 +83,7 @@ $find_changed_cmd = "find \"$srcpathname/\" -not -cmin $minage -a \\( $movie_fi # What command should be used to find all movie files $find_cmd = "find \"$srcpathname/\" $movie_file_ext"; $find_cmd_with_mkv = "find \"$srcpathname/\" $movie_file_ext -o -iregex \".*\.mkv\""; +$find_mkv = "find \"$srcpathname/\" -iregex \".*\.mkv\""; # Set the tmpfile to use, but remove the 0 byte file it creates, we'll create it if we need it $tmpfile = `tempfile`; chomp($tmpfile); unlink "$tmpfile";