From: Alan J. Pippin <alan@pippins.net>
Date: Sat, 29 Oct 2022 17:30:12 +0000 (-0600)
Subject: Fixed various bugs in video copy and merge scripts
X-Git-Url: http://git.pippins.net/embedvideo/static/%27%20%20%20%20.%20%24GLOBALS%5B%27phpgw%27%5D-%3Elink%28%27inc/jquery/%7Bview%7D?a=commitdiff_plain;h=aea4af9869a2086c5085afb34a1bd790940c8b7d;p=videoscripts%2F.git

Fixed various bugs in video copy and merge scripts

Added support to extract video taken date from file vs relying alone on modification time
---

diff --git a/NameMyTVSeries b/NameMyTVSeries
new file mode 100755
index 0000000..fda7b56
Binary files /dev/null and b/NameMyTVSeries differ
diff --git a/aired_today_playlist.py b/aired_today_playlist.py
new file mode 100755
index 0000000..c06e5e8
--- /dev/null
+++ b/aired_today_playlist.py
@@ -0,0 +1,77 @@
+"""
+Create a Plex Playlist with what aired on this day in history (month-day), sort by oldest first.
+If Playlist from yesterday exists delete and create today's.
+If today's Playlist exists exit.
+"""
+
+import operator
+from plexapi.server import PlexServer
+import requests
+import datetime
+
+PLEX_URL = 'http://localhost:32400'
+PLEX_TOKEN = 'xxxxx'
+
+LIBRARY_NAMES = ['Movies', 'TV Shows'] # Your library names
+
+today = datetime.datetime.now().date()
+
+TODAY_PLAY_TITLE = 'Aired Today {}-{}'.format(today.month, today.day)
+
+plex = PlexServer(PLEX_URL, PLEX_TOKEN)
+
+def remove_old():
+    # Remove old Aired Today Playlists
+    for playlist in plex.playlists():
+        if playlist.title.startswith('Aired Today') and playlist.title != TODAY_PLAY_TITLE:
+            playlist.delete()
+            print('Removing old Aired Today Playlists: {}'.format(playlist.title))
+        elif playlist.title == TODAY_PLAY_TITLE:
+            print('{} already exists. No need to make again.'.format(TODAY_PLAY_TITLE))
+            exit(0)
+
+
+def get_all_content(library_name):
+    # Get all movies or episodes from LIBRARY_NAME
+    child_lst = []
+    for library in library_name:
+        for child in plex.library.section(library).all():
+            if child.type == 'movie':
+                child_lst += [child]
+            elif child.type == 'show':
+                child_lst += child.episodes()
+            else:
+                pass
+    return child_lst
+
+
+def find_air_dates(content_lst):
+    # Find what aired with today's month-day
+    aired_lst = []
+    for video in content_lst:
+        try:
+            ad_month = str(video.originallyAvailableAt.month)
+            ad_day = str(video.originallyAvailableAt.day)
+            
+            if ad_month == str(today.month) and ad_day == str(today.day):
+                aired_lst += [[video] + [str(video.originallyAvailableAt)]]
+        except Exception as e:
+            # print(e)
+            pass
+        
+        # Sort by original air date, oldest first
+        aired_lst = sorted(aired_lst, key=operator.itemgetter(1))
+
+    # Remove date used for sorting
+    play_lst = [x[0] for x in aired_lst]
+    return play_lst
+
+
+remove_old()
+play_lst = find_air_dates(get_all_content(LIBRARY_NAMES))
+# Create Playlist
+if play_lst:
+    plex.createPlaylist(TODAY_PLAY_TITLE, play_lst)
+else:
+    print('Found nothing aired on this day in history.')
+
diff --git a/check_phone_videos_are_in_originals b/check_phone_videos_are_in_originals
new file mode 100755
index 0000000..f002e59
--- /dev/null
+++ b/check_phone_videos_are_in_originals
@@ -0,0 +1,24 @@
+#!/bin/bash
+# Checks to make sure all videos off of phone have been copied to the Originals directory
+# If they are missing, it means Nextcloud missed them somehow. Copy them to New Memories then.
+
+dcim_path="/storage/emulated/0/DCIM/Camera"
+originals="/naspool/dropbox/Originals"
+new_memories="/naspool/dropbox/New Memories"
+
+adb connect 192.168.0.23
+if [[ $? != 0 ]]; then
+  echo "#ERROR: Unable to connect to phone at 192.168.0.23"
+  exit 1
+fi
+
+mp4s_on_phone=$(adb -e shell ls -1 $dcim_path/*.mp4 | xargs -n 1 basename)
+mp4s_in_originals=$(/bin/ls -1 $originals/*.mp4 | xargs -n 1 basename)
+
+for i in $mp4s_on_phone; do
+  echo "$mp4s_in_originals" | grep -q $i
+  if [[ $? == 1 ]]; then
+    echo "mp4 not found in originals: $i"
+    adb -e pull "$dcim_path/$i" "$new_memories"
+  fi
+done
diff --git a/ffprobe b/ffprobe
new file mode 100755
index 0000000..9d0198b
Binary files /dev/null and b/ffprobe differ
diff --git a/make_mkv b/make_mkv
index bf98325..f048116 100755
--- a/make_mkv
+++ b/make_mkv
@@ -17,6 +17,10 @@ use Data::Dumper;
 use DateTime;
 use DateTime::Duration;
 use DateTime::Format::Duration;
+use DateTime::Format::Strptime qw( );
+
+# Set our ffmpeg creation_time format
+$ffmpeg_time_format = DateTime::Format::Strptime->new(pattern=>'%Y-%m-%dT%H:%M:%S', time_zone => 'UTC', on_error => 'croak');
 
 ####################################################################################################
 # Configuration parameters
@@ -66,7 +70,15 @@ sub epoch_to_date {
 my %videos;
 foreach $video (split(/,/, $opt_i)) {
     if(! -r "$video") { die "-E- Unable to read input video file: $video\n"; }
-    my $mtime_epoch = stat("$video")->mtime;
+    my $mtime_epoch = 0;
+    my $creation_time = `$ffmpeg -i "$video" 2>&1 | grep "creation_time" | head -n 1 | awk '{print \$3}'`;
+    if($creation_time) {
+	my $date_taken = $ffmpeg_time_format->parse_datetime($creation_time);
+	$date_taken->set_time_zone('local');
+	$mtime_epoch = $date_taken->epoch;
+    } else {
+	$mtime_epoch = stat("$video")->mtime;
+    }
     my $mdate = epoch_to_date($mtime_epoch);
     $videos{$video} = $mtime_epoch;
 }
@@ -191,6 +203,10 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) {
 		    unlink "$video_mp4";
 		    die "-E- handbrake encountered some errors with exit code $errno\n";
 		}
+		# Restore the metadata from the original video to the intermediate video to preserve the creation_time of the original video metadata in it
+		`$ffmpeg -i \"$video\" -i \"$video_mp4\" -map 1 -map_metadata 0 -c copy -movflags use_metadata_tags \"$video_mp4.fixed.mp4\" >> \"$tmpfile\" 2>&1`;
+		$errno=system("mv \"$video_mp4.fixed.mp4\" \"$video_mp4\" 2>/dev/null");
+		if($errno) { print "-E- Error moving fixed metadata video back to dstfile: $video_mp4.fixed.mp4 -> $video_mp4\n"; }
 	    }
 	} else {
 	    if($no_recompress) { 
@@ -273,7 +289,15 @@ foreach my $key (keys %merge_videos) {
 	my $hour = 0;
 	my $min  = 5;
 	my $sec  = 0;
-	my $mtime_epoch = stat("$video")->mtime;
+	my $mtime_epoch = 0;
+	my $creation_time = `$ffmpeg -i "$video" 2>&1 | grep "creation_time" | head -n 1 | awk '{print \$3}'`;
+	if($creation_time) {
+	    my $date_taken = $ffmpeg_time_format->parse_datetime($creation_time);
+	    $date_taken->set_time_zone('local');
+	    $mtime_epoch = $date_taken->epoch;
+	} else {
+	    $mtime_epoch = stat("$video")->mtime;
+	}
 	my $mdate = epoch_to_date($mtime_epoch);
 	my $duration = `$ffmpeg -i "$video" 2>&1 | grep Duration`;
 	if($duration =~ /Duration: (\d+):(\d+):(\d+)\.(\d+)/) {
@@ -309,7 +333,7 @@ foreach my $key (keys %merge_videos) {
     }
     print "\n$merge_cmd\n";
     if(! defined $opt_s) {
-	if(-f "$opt_o.$key.mkv") { die "-E- Output video file $opt_o already exists. This is unexpected for key $key!"; }
+	#if(-f "$opt_o.$key.mkv") { die "-E- Output video file $opt_o already exists. This is unexpected for key $key!"; }
 	my $errno = system("$merge_cmd");
 	$errno = $errno >> 8;
 	if($errno > 1) {
diff --git a/merge_videos_by_day b/merge_videos_by_day
index 146ac84..193a7bd 100755
--- a/merge_videos_by_day
+++ b/merge_videos_by_day
@@ -7,6 +7,11 @@ use File::Basename;
 use Getopt::Std;
 use File::stat;
 use Time::localtime;
+use DateTime::Format::Strptime qw( );
+
+# Set our ffmpeg creation_time format
+$ffmpeg_time_format = DateTime::Format::Strptime->new(pattern=>'%Y-%m-%dT%H:%M:%S', time_zone => 'UTC', on_error => 'croak');
+$ctime_format = DateTime::Format::Strptime->new(pattern=>'%a %b %d %H:%M:%S %Y', time_zone => 'local', on_error => 'croak');
 
 # Early command line options processing
 getopts("qzkh:tvs:");
@@ -96,12 +101,28 @@ foreach $file (sort `$find_cmd`) {
     if($srcfile =~ /.hb.mp4/) { next; }
 
     print "Found video: srcdir: $srcdir srcfile: $srcfile srcext: $srcext dstext: $ext\n" if($opt_v);
-            
-    # Make a note of the month, year, and day this video was taken (from the modification time of the file)
-    $date_taken = ctime(stat("$srcdir/$srcfile")->mtime);
-    
+
+    # Make a note of the month, year, and day this video was taken
+    $creation_time = `$ffmpeg -i "$srcdir/$srcfile" 2>&1 | grep "creation_time" | head -n 1 | awk '{print \$3}'`;
+    if($creation_time) {
+	$date_taken = $ffmpeg_time_format->parse_datetime($creation_time);
+    } else {
+	# From the modification time of the file since we couldn't find it in the video file
+	$date_modified = ctime(stat("$srcdir/$srcfile")->mtime);
+    }
+
+    # Get the date taken from the ffmpeg creation_time
+    if(!$merge_by_modification_time && $date_taken) {
+	$date_taken->set_time_zone('local');
+	$year = $date_taken->year;
+	$month = sprintf("%02d",$date_taken->month);
+	$day = sprintf("%02d",$date_taken->day);
+	$monthnum = sprintf("%02d",$date_taken->month);
+	$monthname = lc($month2monthname{$month});
+	#print "date_taken: $year-$month-$day ".$date_taken->hour.":".$date_taken->minute.":".$date_taken->second." ".$date_taken->time_zone."\n";
+    }
     # Get the date taken from the filename
-    if(!$merge_by_modification_date && $srcfile =~ /^(\d\d\d\d)-(\d\d)-(\d\d)/) {
+    elsif(!$merge_by_modification_date && $srcfile =~ /^(\d\d\d\d)-(\d\d)-(\d\d)/) {
 	$year = $1;
 	$month = $2;
 	$day = sprintf("%02d",$3);
@@ -124,7 +145,7 @@ foreach $file (sort `$find_cmd`) {
 	$monthname = lc($month2monthname{$month});
     }
     # Get the date taken from the modification time
-    elsif($date_taken =~ /\S+\s+(\S+)\s+(\d+)\s+\S+\s+(\d+)/) {
+    elsif($date_modified =~ /\S+\s+(\S+)\s+(\d+)\s+\S+\s+(\d+)/) {
 	$year = $3;
 	$month = $1;
 	$day = sprintf("%02d",$2);
diff --git a/mkv2dvd b/mkv2dvd
new file mode 100755
index 0000000..cc07c43
--- /dev/null
+++ b/mkv2dvd
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+SRC=$1
+NAME="${SRC%.*}"
+MPG="${NAME}.mpg"
+DVD="${NAME}.dvd"
+ISO="${NAME}.iso"
+
+export VIDEO_FORMAT=NTSC
+
+if [[ -z $SRC ]] || [[ ! -r $SRC ]]; then
+    echo "-E- You need to specify a valid src video file as an argument"
+    exit 1
+fi
+
+if [[ ! -e "$MPG" ]]; then
+    echo "-> Converting \"$SRC\" -> \"$MPG\""
+    # Convert the SRC video into a DVD compatible mpg2 file
+    ffmpeg -i "$SRC" -target ntsc-dvd -aspect 16:9 "$MPG"
+    if [[ $? != 0 ]]; then
+	echo "-> ffmpeg encountered an error. aborting..."
+	rm "$MPG"
+	exit 1;
+    fi
+fi
+
+echo "-> Creating DVD ISO: $ISO"
+# Create an intermediate DVD dir
+mkdir -p "$DVD"
+# Define the title file
+dvdauthor -o "$DVD" -t "$MPG"
+# Create a table of contents
+dvdauthor -o "$DVD" -T
+# Convert the DVD dir into an ISO image
+genisoimage -dvd-video -o "$ISO" "$DVD"
+# Remove the intermediate DVD dir
+rm -rf "$DVD"
+# Remove the intermediate MPG file
+rm "$MPG"
diff --git a/move_videos_from_watchdir b/move_videos_from_watchdir
index 364ade6..864d33e 100755
--- a/move_videos_from_watchdir
+++ b/move_videos_from_watchdir
@@ -85,7 +85,7 @@ foreach $watchpath (split(';', $watchpathname)) {
         } 
 
 	# Make sure we have a unique dstfile
-	if(-f "$dstfile") {
+	if($overwrite_dest == 0 && -f "$dstfile") {
 	    $video_ext = $srcfile;
 	    $video_ext =~ s/.*\.(\S+)$/$1/;
 	    $suffix = 0;
@@ -102,18 +102,22 @@ foreach $watchpath (split(';', $watchpathname)) {
 
 	if(!defined $opt_t) {
 	    # Make sure the dstfile doesn't exist, if it does, add a unique number to the end
-	    if(! -f "$dstfile") {
-		$errno=system("mv \"$srcdir/$srcfile\" \"$dstfile\" 2>/dev/null");
-		if($errno) { print "-E- Error moving srcfile to dstfile: $srcdir/$srcfile -> $dstfile\n"; next; }
-	    } else {
-		print "-E- Unable to mv $srcdir/$srcfile -> $dstfile because it already exists\n";
-	    }
+	    #if(! -f "$dstfile") {
+	    $errno=system("mv \"$srcdir/$srcfile\" \"$dstfile\" 2>/dev/null");
+	    if($errno) { print "-E- Error moving srcfile to dstfile: $srcdir/$srcfile -> $dstfile\n"; next; }
+	    #} else {
+	    #	print "-E- Unable to mv $srcdir/$srcfile -> $dstfile because it already exists\n";
+	    #}
 	    # Fix the permissions
 	    system("chown $owner \"$dstfile\"");
 	    system("chgrp $group \"$dstfile\"");
 	    system("chmod $mode \"$dstfile\"");
 	}
     }
+
+    # Update nextcloud file cache so it knows what files we have moved
+    $watchpath =~ s/$nextcloudbase//g;
+    system("$occ files:scan --path \"$watchpath\" > /dev/null");
 }
 		    
 $pidfile->remove();
diff --git a/organize_videos b/organize_videos
index 05edab4..d62af19 100755
--- a/organize_videos
+++ b/organize_videos
@@ -179,10 +179,13 @@ foreach $file (`$find_mkv`) {
     print "srcfile: $srcfile\n";
     if($srcfile =~ /.hb.mp4/) { next; }
     
-    print "Found movie: srcdir: $srcdir srcfile: $srcfile ext: $ext\n" if($opt_v);
-        
-    # Make a note of the month, year, and day this video was taken (from the modification time of the file)
-    $date_taken = ctime(stat("$srcdir/$srcfile")->mtime);
+    print "Found video: srcdir: $srcdir srcfile: $srcfile ext: $ext\n" if($opt_v);
+
+    # From the modification time of the file since we couldn't find it in the filename
+    $date_modified = ctime(stat("$srcdir/$srcfile")->mtime);
+
+    # NOTE: This file matching algorithm only applies to videos produced by merge_videos_by_day called by this script earlier
+    # We just need to take those merged videos, and extract what date to call the video by under HomeVideos
 
     # Get the date taken from the filename
     if($srcfile =~ /^(\d+)-(\d+)-(\d+)/) {
@@ -208,7 +211,7 @@ foreach $file (`$find_mkv`) {
 	$monthname = lc($month2monthname{$month});
     }
     # Get the date taken from the modification time
-    elsif($date_taken =~ /\S+\s+(\S+)\s+(\d+)\s+\S+\s+(\d+)/) {
+    elsif($date_modified =~ /\S+\s+(\S+)\s+(\d+)\s+\S+\s+(\d+)/) {
 	$year = $3;
 	$month = $1;
 	$day = sprintf("%02d",$2);
diff --git a/organize_videos.conf b/organize_videos.conf
index fd12413..27b7933 100644
--- a/organize_videos.conf
+++ b/organize_videos.conf
@@ -26,11 +26,20 @@ $save_originals = 1;
 # Path to a dir (or dirs separated by semis) to watch for videos to move to $srcpathname to be organized
 $watchpathname = "/naspool/cloud/alan/files/InstantUpload/Camera;/naspool/cloud/mary/files/InstantUpload/Camera;";
 
+# Path to nextcloud base dir where user dirs start
+$nextcloudbase = "/naspool/cloud/";
+
+# If there are file duplicates in srcpathname, should they be overwritten (assumes all files from phones will be named uniquely)
+$overwrite_dest = 1;
+
+# Path to nextcloud occ cmd script
+$occ = "/naspool/www/pippins.net/nextcloud.custom/occ.cmd";
+
 # Path to merge_videos_by_day script
 $merge_videos_by_day = "/naspool/videos/bin/merge_videos_by_day";
 
 # Flag to merge videos by modification date instead of the date parsed from the filename
-$merge_by_modification_date = 1;
+$merge_by_modification_date = 0;
 
 # Path to the make_mkv script
 $make_mkv = "/naspool/videos/bin/make_mkv";
@@ -76,7 +85,7 @@ $no_recompress_file_ext = qr/\.(mp4)$/i;
 
 # Video file creation dates must not have changed in the last X minutes to process any of the video files
 # This is done to ensure that all videos from a given upload from a camera have completed prior to looking for videos to merge
-$minage = "+120";
+$minage = "+60";
 
 # What command should be used to find files that have changed (are at least $minage old) 
 $find_changed_cmd = "find  \"$srcpathname/\" -not -cmin $minage -a \\( $movie_file_ext \\)";
diff --git a/remove_empty_tv_dirs b/remove_empty_tv_dirs
new file mode 100755
index 0000000..b9b7941
--- /dev/null
+++ b/remove_empty_tv_dirs
@@ -0,0 +1,79 @@
+#!/usr/bin/perl
+# Author: Alan J. Pippin
+# Description: Find and remove TV recording directories that don't contain any recordings in them
+
+use File::Copy;
+use File::Basename;
+use Getopt::Std;
+use File::stat;
+use Time::localtime;
+use File::Pid;
+use Data::Dumper;
+
+####################################################################################################
+# Configuration parameters
+my $tv_src_dir = "/tvpool/tv";
+my $log = "/var/log/organize/remove_empty_tv_dirs.log";
+my $tv_file_ext = "-iregex \".*\.mov\" -o -iregex \".*\.avi\" -o -iregex \".*\.mp4\" -o -iregex \".*\.mpg\" -o -iregex \".*\.ts\" -o -iregex \".*\.mkv\"";
+my $find_tv_show_dirs = "find \"$tv_src_dir/\" -maxdepth 1 -cmin +14400 -type d";
+my $find_tv_season_dirs = "find \"$tv_src_dir/\" -maxdepth 2 -mindepth 2 -cmin +14400 -type d";
+my $find_tv_shows = "find \"$tv_src_dir/\" -cmin +14400 $tv_file_ext";
+
+####################################################################################################
+# Find empty TV show directories to remove
+
+my %tv_show_dirs;
+
+# Find all of our TV season dirs and store them in a hash
+foreach $dir (`$find_tv_season_dirs`) {
+    chomp($dir);
+    my $season = basename($dir);
+    my $show = basename(dirname($dir));
+    #print "-> Found dir $dir show $show season $season\n";
+    $tv_show_dirs{$show}{$season}{has_recordings} = 0;
+}
+foreach $dir (`$find_tv_show_dirs`) {
+    chomp($dir);
+    my $show = basename($dir);
+    #print "-> Found dir $dir show $show\n";
+    $tv_show_dirs{$show}{has_recordings} = 0;
+}
+
+# Find all of our TV show recordings and store them in a hash
+foreach $file (`$find_tv_shows`) {
+    chomp($file);
+    my $recording = basename($file);
+    my $season = basename(dirname($file));
+    my $show = basename(dirname(dirname($file)));
+    #print "-> Found show $show season $season and recording $recording\n";
+    $tv_show_dirs{$show}{$season}{has_recordings} = 1;
+    $tv_show_dirs{$show}{has_recordings} = 1;
+}
+
+#print Dumper(\%tv_show_dirs);
+
+# For each season directory that doesn't have recordings, remove it
+foreach $show (keys %tv_show_dirs) {
+    foreach $season (keys %{$tv_show_dirs{$show}}) {
+	if($season eq "has_recordings") { next; }
+	if ($tv_show_dirs{$show}{$season}{has_recordings} == 0) {
+	    print "-> Removing season dir '$show/$season'\n";
+	    system("rm -rf \"$tv_src_dir/$show/$season/\"");
+	}
+    }
+}
+
+# For each show directory that doesn't have any recordings under it, remove it
+foreach $show (keys %tv_show_dirs) {
+    my $has_recordings = 0;
+    if($show eq ".grab") { next; }
+    foreach $season (keys %{$tv_show_dirs{$show}}) {
+	if ($tv_show_dirs{$show}{$season}{has_recordings} == 1) {
+	    $has_recordings = 1;
+	}
+    }
+    if($has_recordings == 0) { 
+	print "-> Removing show dir '$show'\n";
+	system("rm -rf \"$tv_src_dir/$show/\"");
+    }
+}