From: Alan J. Pippin Date: Sat, 29 Oct 2022 17:30:12 +0000 (-0600) Subject: Fixed various bugs in video copy and merge scripts X-Git-Url: http://git.pippins.net/embedvideo/.git/%27%20%20%20%20.%20%24GLOBALS%5B%27phpgw%27%5D-%3Elink%28%27inc/jquery/%22%22.%24thumbnail.%22/%7Blink_attendance%7D?a=commitdiff_plain;h=aea4af9869a2086c5085afb34a1bd790940c8b7d;p=videoscripts%2F.git Fixed various bugs in video copy and merge scripts Added support to extract video taken date from file vs relying alone on modification time --- diff --git a/NameMyTVSeries b/NameMyTVSeries new file mode 100755 index 0000000..fda7b56 Binary files /dev/null and b/NameMyTVSeries differ diff --git a/aired_today_playlist.py b/aired_today_playlist.py new file mode 100755 index 0000000..c06e5e8 --- /dev/null +++ b/aired_today_playlist.py @@ -0,0 +1,77 @@ +""" +Create a Plex Playlist with what aired on this day in history (month-day), sort by oldest first. +If Playlist from yesterday exists delete and create today's. +If today's Playlist exists exit. +""" + +import operator +from plexapi.server import PlexServer +import requests +import datetime + +PLEX_URL = 'http://localhost:32400' +PLEX_TOKEN = 'xxxxx' + +LIBRARY_NAMES = ['Movies', 'TV Shows'] # Your library names + +today = datetime.datetime.now().date() + +TODAY_PLAY_TITLE = 'Aired Today {}-{}'.format(today.month, today.day) + +plex = PlexServer(PLEX_URL, PLEX_TOKEN) + +def remove_old(): + # Remove old Aired Today Playlists + for playlist in plex.playlists(): + if playlist.title.startswith('Aired Today') and playlist.title != TODAY_PLAY_TITLE: + playlist.delete() + print('Removing old Aired Today Playlists: {}'.format(playlist.title)) + elif playlist.title == TODAY_PLAY_TITLE: + print('{} already exists. No need to make again.'.format(TODAY_PLAY_TITLE)) + exit(0) + + +def get_all_content(library_name): + # Get all movies or episodes from LIBRARY_NAME + child_lst = [] + for library in library_name: + for child in plex.library.section(library).all(): + if child.type == 'movie': + child_lst += [child] + elif child.type == 'show': + child_lst += child.episodes() + else: + pass + return child_lst + + +def find_air_dates(content_lst): + # Find what aired with today's month-day + aired_lst = [] + for video in content_lst: + try: + ad_month = str(video.originallyAvailableAt.month) + ad_day = str(video.originallyAvailableAt.day) + + if ad_month == str(today.month) and ad_day == str(today.day): + aired_lst += [[video] + [str(video.originallyAvailableAt)]] + except Exception as e: + # print(e) + pass + + # Sort by original air date, oldest first + aired_lst = sorted(aired_lst, key=operator.itemgetter(1)) + + # Remove date used for sorting + play_lst = [x[0] for x in aired_lst] + return play_lst + + +remove_old() +play_lst = find_air_dates(get_all_content(LIBRARY_NAMES)) +# Create Playlist +if play_lst: + plex.createPlaylist(TODAY_PLAY_TITLE, play_lst) +else: + print('Found nothing aired on this day in history.') + diff --git a/check_phone_videos_are_in_originals b/check_phone_videos_are_in_originals new file mode 100755 index 0000000..f002e59 --- /dev/null +++ b/check_phone_videos_are_in_originals @@ -0,0 +1,24 @@ +#!/bin/bash +# Checks to make sure all videos off of phone have been copied to the Originals directory +# If they are missing, it means Nextcloud missed them somehow. Copy them to New Memories then. + +dcim_path="/storage/emulated/0/DCIM/Camera" +originals="/naspool/dropbox/Originals" +new_memories="/naspool/dropbox/New Memories" + +adb connect 192.168.0.23 +if [[ $? != 0 ]]; then + echo "#ERROR: Unable to connect to phone at 192.168.0.23" + exit 1 +fi + +mp4s_on_phone=$(adb -e shell ls -1 $dcim_path/*.mp4 | xargs -n 1 basename) +mp4s_in_originals=$(/bin/ls -1 $originals/*.mp4 | xargs -n 1 basename) + +for i in $mp4s_on_phone; do + echo "$mp4s_in_originals" | grep -q $i + if [[ $? == 1 ]]; then + echo "mp4 not found in originals: $i" + adb -e pull "$dcim_path/$i" "$new_memories" + fi +done diff --git a/ffprobe b/ffprobe new file mode 100755 index 0000000..9d0198b Binary files /dev/null and b/ffprobe differ diff --git a/make_mkv b/make_mkv index bf98325..f048116 100755 --- a/make_mkv +++ b/make_mkv @@ -17,6 +17,10 @@ use Data::Dumper; use DateTime; use DateTime::Duration; use DateTime::Format::Duration; +use DateTime::Format::Strptime qw( ); + +# Set our ffmpeg creation_time format +$ffmpeg_time_format = DateTime::Format::Strptime->new(pattern=>'%Y-%m-%dT%H:%M:%S', time_zone => 'UTC', on_error => 'croak'); #################################################################################################### # Configuration parameters @@ -66,7 +70,15 @@ sub epoch_to_date { my %videos; foreach $video (split(/,/, $opt_i)) { if(! -r "$video") { die "-E- Unable to read input video file: $video\n"; } - my $mtime_epoch = stat("$video")->mtime; + my $mtime_epoch = 0; + my $creation_time = `$ffmpeg -i "$video" 2>&1 | grep "creation_time" | head -n 1 | awk '{print \$3}'`; + if($creation_time) { + my $date_taken = $ffmpeg_time_format->parse_datetime($creation_time); + $date_taken->set_time_zone('local'); + $mtime_epoch = $date_taken->epoch; + } else { + $mtime_epoch = stat("$video")->mtime; + } my $mdate = epoch_to_date($mtime_epoch); $videos{$video} = $mtime_epoch; } @@ -191,6 +203,10 @@ foreach my $video (sort{$videos{$a} <=> $videos{$b}} keys %videos) { unlink "$video_mp4"; die "-E- handbrake encountered some errors with exit code $errno\n"; } + # Restore the metadata from the original video to the intermediate video to preserve the creation_time of the original video metadata in it + `$ffmpeg -i \"$video\" -i \"$video_mp4\" -map 1 -map_metadata 0 -c copy -movflags use_metadata_tags \"$video_mp4.fixed.mp4\" >> \"$tmpfile\" 2>&1`; + $errno=system("mv \"$video_mp4.fixed.mp4\" \"$video_mp4\" 2>/dev/null"); + if($errno) { print "-E- Error moving fixed metadata video back to dstfile: $video_mp4.fixed.mp4 -> $video_mp4\n"; } } } else { if($no_recompress) { @@ -273,7 +289,15 @@ foreach my $key (keys %merge_videos) { my $hour = 0; my $min = 5; my $sec = 0; - my $mtime_epoch = stat("$video")->mtime; + my $mtime_epoch = 0; + my $creation_time = `$ffmpeg -i "$video" 2>&1 | grep "creation_time" | head -n 1 | awk '{print \$3}'`; + if($creation_time) { + my $date_taken = $ffmpeg_time_format->parse_datetime($creation_time); + $date_taken->set_time_zone('local'); + $mtime_epoch = $date_taken->epoch; + } else { + $mtime_epoch = stat("$video")->mtime; + } my $mdate = epoch_to_date($mtime_epoch); my $duration = `$ffmpeg -i "$video" 2>&1 | grep Duration`; if($duration =~ /Duration: (\d+):(\d+):(\d+)\.(\d+)/) { @@ -309,7 +333,7 @@ foreach my $key (keys %merge_videos) { } print "\n$merge_cmd\n"; if(! defined $opt_s) { - if(-f "$opt_o.$key.mkv") { die "-E- Output video file $opt_o already exists. This is unexpected for key $key!"; } + #if(-f "$opt_o.$key.mkv") { die "-E- Output video file $opt_o already exists. This is unexpected for key $key!"; } my $errno = system("$merge_cmd"); $errno = $errno >> 8; if($errno > 1) { diff --git a/merge_videos_by_day b/merge_videos_by_day index 146ac84..193a7bd 100755 --- a/merge_videos_by_day +++ b/merge_videos_by_day @@ -7,6 +7,11 @@ use File::Basename; use Getopt::Std; use File::stat; use Time::localtime; +use DateTime::Format::Strptime qw( ); + +# Set our ffmpeg creation_time format +$ffmpeg_time_format = DateTime::Format::Strptime->new(pattern=>'%Y-%m-%dT%H:%M:%S', time_zone => 'UTC', on_error => 'croak'); +$ctime_format = DateTime::Format::Strptime->new(pattern=>'%a %b %d %H:%M:%S %Y', time_zone => 'local', on_error => 'croak'); # Early command line options processing getopts("qzkh:tvs:"); @@ -96,12 +101,28 @@ foreach $file (sort `$find_cmd`) { if($srcfile =~ /.hb.mp4/) { next; } print "Found video: srcdir: $srcdir srcfile: $srcfile srcext: $srcext dstext: $ext\n" if($opt_v); - - # Make a note of the month, year, and day this video was taken (from the modification time of the file) - $date_taken = ctime(stat("$srcdir/$srcfile")->mtime); - + + # Make a note of the month, year, and day this video was taken + $creation_time = `$ffmpeg -i "$srcdir/$srcfile" 2>&1 | grep "creation_time" | head -n 1 | awk '{print \$3}'`; + if($creation_time) { + $date_taken = $ffmpeg_time_format->parse_datetime($creation_time); + } else { + # From the modification time of the file since we couldn't find it in the video file + $date_modified = ctime(stat("$srcdir/$srcfile")->mtime); + } + + # Get the date taken from the ffmpeg creation_time + if(!$merge_by_modification_time && $date_taken) { + $date_taken->set_time_zone('local'); + $year = $date_taken->year; + $month = sprintf("%02d",$date_taken->month); + $day = sprintf("%02d",$date_taken->day); + $monthnum = sprintf("%02d",$date_taken->month); + $monthname = lc($month2monthname{$month}); + #print "date_taken: $year-$month-$day ".$date_taken->hour.":".$date_taken->minute.":".$date_taken->second." ".$date_taken->time_zone."\n"; + } # Get the date taken from the filename - if(!$merge_by_modification_date && $srcfile =~ /^(\d\d\d\d)-(\d\d)-(\d\d)/) { + elsif(!$merge_by_modification_date && $srcfile =~ /^(\d\d\d\d)-(\d\d)-(\d\d)/) { $year = $1; $month = $2; $day = sprintf("%02d",$3); @@ -124,7 +145,7 @@ foreach $file (sort `$find_cmd`) { $monthname = lc($month2monthname{$month}); } # Get the date taken from the modification time - elsif($date_taken =~ /\S+\s+(\S+)\s+(\d+)\s+\S+\s+(\d+)/) { + elsif($date_modified =~ /\S+\s+(\S+)\s+(\d+)\s+\S+\s+(\d+)/) { $year = $3; $month = $1; $day = sprintf("%02d",$2); diff --git a/mkv2dvd b/mkv2dvd new file mode 100755 index 0000000..cc07c43 --- /dev/null +++ b/mkv2dvd @@ -0,0 +1,39 @@ +#!/bin/bash + +SRC=$1 +NAME="${SRC%.*}" +MPG="${NAME}.mpg" +DVD="${NAME}.dvd" +ISO="${NAME}.iso" + +export VIDEO_FORMAT=NTSC + +if [[ -z $SRC ]] || [[ ! -r $SRC ]]; then + echo "-E- You need to specify a valid src video file as an argument" + exit 1 +fi + +if [[ ! -e "$MPG" ]]; then + echo "-> Converting \"$SRC\" -> \"$MPG\"" + # Convert the SRC video into a DVD compatible mpg2 file + ffmpeg -i "$SRC" -target ntsc-dvd -aspect 16:9 "$MPG" + if [[ $? != 0 ]]; then + echo "-> ffmpeg encountered an error. aborting..." + rm "$MPG" + exit 1; + fi +fi + +echo "-> Creating DVD ISO: $ISO" +# Create an intermediate DVD dir +mkdir -p "$DVD" +# Define the title file +dvdauthor -o "$DVD" -t "$MPG" +# Create a table of contents +dvdauthor -o "$DVD" -T +# Convert the DVD dir into an ISO image +genisoimage -dvd-video -o "$ISO" "$DVD" +# Remove the intermediate DVD dir +rm -rf "$DVD" +# Remove the intermediate MPG file +rm "$MPG" diff --git a/move_videos_from_watchdir b/move_videos_from_watchdir index 364ade6..864d33e 100755 --- a/move_videos_from_watchdir +++ b/move_videos_from_watchdir @@ -85,7 +85,7 @@ foreach $watchpath (split(';', $watchpathname)) { } # Make sure we have a unique dstfile - if(-f "$dstfile") { + if($overwrite_dest == 0 && -f "$dstfile") { $video_ext = $srcfile; $video_ext =~ s/.*\.(\S+)$/$1/; $suffix = 0; @@ -102,18 +102,22 @@ foreach $watchpath (split(';', $watchpathname)) { if(!defined $opt_t) { # Make sure the dstfile doesn't exist, if it does, add a unique number to the end - if(! -f "$dstfile") { - $errno=system("mv \"$srcdir/$srcfile\" \"$dstfile\" 2>/dev/null"); - if($errno) { print "-E- Error moving srcfile to dstfile: $srcdir/$srcfile -> $dstfile\n"; next; } - } else { - print "-E- Unable to mv $srcdir/$srcfile -> $dstfile because it already exists\n"; - } + #if(! -f "$dstfile") { + $errno=system("mv \"$srcdir/$srcfile\" \"$dstfile\" 2>/dev/null"); + if($errno) { print "-E- Error moving srcfile to dstfile: $srcdir/$srcfile -> $dstfile\n"; next; } + #} else { + # print "-E- Unable to mv $srcdir/$srcfile -> $dstfile because it already exists\n"; + #} # Fix the permissions system("chown $owner \"$dstfile\""); system("chgrp $group \"$dstfile\""); system("chmod $mode \"$dstfile\""); } } + + # Update nextcloud file cache so it knows what files we have moved + $watchpath =~ s/$nextcloudbase//g; + system("$occ files:scan --path \"$watchpath\" > /dev/null"); } $pidfile->remove(); diff --git a/organize_videos b/organize_videos index 05edab4..d62af19 100755 --- a/organize_videos +++ b/organize_videos @@ -179,10 +179,13 @@ foreach $file (`$find_mkv`) { print "srcfile: $srcfile\n"; if($srcfile =~ /.hb.mp4/) { next; } - print "Found movie: srcdir: $srcdir srcfile: $srcfile ext: $ext\n" if($opt_v); - - # Make a note of the month, year, and day this video was taken (from the modification time of the file) - $date_taken = ctime(stat("$srcdir/$srcfile")->mtime); + print "Found video: srcdir: $srcdir srcfile: $srcfile ext: $ext\n" if($opt_v); + + # From the modification time of the file since we couldn't find it in the filename + $date_modified = ctime(stat("$srcdir/$srcfile")->mtime); + + # NOTE: This file matching algorithm only applies to videos produced by merge_videos_by_day called by this script earlier + # We just need to take those merged videos, and extract what date to call the video by under HomeVideos # Get the date taken from the filename if($srcfile =~ /^(\d+)-(\d+)-(\d+)/) { @@ -208,7 +211,7 @@ foreach $file (`$find_mkv`) { $monthname = lc($month2monthname{$month}); } # Get the date taken from the modification time - elsif($date_taken =~ /\S+\s+(\S+)\s+(\d+)\s+\S+\s+(\d+)/) { + elsif($date_modified =~ /\S+\s+(\S+)\s+(\d+)\s+\S+\s+(\d+)/) { $year = $3; $month = $1; $day = sprintf("%02d",$2); diff --git a/organize_videos.conf b/organize_videos.conf index fd12413..27b7933 100644 --- a/organize_videos.conf +++ b/organize_videos.conf @@ -26,11 +26,20 @@ $save_originals = 1; # Path to a dir (or dirs separated by semis) to watch for videos to move to $srcpathname to be organized $watchpathname = "/naspool/cloud/alan/files/InstantUpload/Camera;/naspool/cloud/mary/files/InstantUpload/Camera;"; +# Path to nextcloud base dir where user dirs start +$nextcloudbase = "/naspool/cloud/"; + +# If there are file duplicates in srcpathname, should they be overwritten (assumes all files from phones will be named uniquely) +$overwrite_dest = 1; + +# Path to nextcloud occ cmd script +$occ = "/naspool/www/pippins.net/nextcloud.custom/occ.cmd"; + # Path to merge_videos_by_day script $merge_videos_by_day = "/naspool/videos/bin/merge_videos_by_day"; # Flag to merge videos by modification date instead of the date parsed from the filename -$merge_by_modification_date = 1; +$merge_by_modification_date = 0; # Path to the make_mkv script $make_mkv = "/naspool/videos/bin/make_mkv"; @@ -76,7 +85,7 @@ $no_recompress_file_ext = qr/\.(mp4)$/i; # Video file creation dates must not have changed in the last X minutes to process any of the video files # This is done to ensure that all videos from a given upload from a camera have completed prior to looking for videos to merge -$minage = "+120"; +$minage = "+60"; # What command should be used to find files that have changed (are at least $minage old) $find_changed_cmd = "find \"$srcpathname/\" -not -cmin $minage -a \\( $movie_file_ext \\)"; diff --git a/remove_empty_tv_dirs b/remove_empty_tv_dirs new file mode 100755 index 0000000..b9b7941 --- /dev/null +++ b/remove_empty_tv_dirs @@ -0,0 +1,79 @@ +#!/usr/bin/perl +# Author: Alan J. Pippin +# Description: Find and remove TV recording directories that don't contain any recordings in them + +use File::Copy; +use File::Basename; +use Getopt::Std; +use File::stat; +use Time::localtime; +use File::Pid; +use Data::Dumper; + +#################################################################################################### +# Configuration parameters +my $tv_src_dir = "/tvpool/tv"; +my $log = "/var/log/organize/remove_empty_tv_dirs.log"; +my $tv_file_ext = "-iregex \".*\.mov\" -o -iregex \".*\.avi\" -o -iregex \".*\.mp4\" -o -iregex \".*\.mpg\" -o -iregex \".*\.ts\" -o -iregex \".*\.mkv\""; +my $find_tv_show_dirs = "find \"$tv_src_dir/\" -maxdepth 1 -cmin +14400 -type d"; +my $find_tv_season_dirs = "find \"$tv_src_dir/\" -maxdepth 2 -mindepth 2 -cmin +14400 -type d"; +my $find_tv_shows = "find \"$tv_src_dir/\" -cmin +14400 $tv_file_ext"; + +#################################################################################################### +# Find empty TV show directories to remove + +my %tv_show_dirs; + +# Find all of our TV season dirs and store them in a hash +foreach $dir (`$find_tv_season_dirs`) { + chomp($dir); + my $season = basename($dir); + my $show = basename(dirname($dir)); + #print "-> Found dir $dir show $show season $season\n"; + $tv_show_dirs{$show}{$season}{has_recordings} = 0; +} +foreach $dir (`$find_tv_show_dirs`) { + chomp($dir); + my $show = basename($dir); + #print "-> Found dir $dir show $show\n"; + $tv_show_dirs{$show}{has_recordings} = 0; +} + +# Find all of our TV show recordings and store them in a hash +foreach $file (`$find_tv_shows`) { + chomp($file); + my $recording = basename($file); + my $season = basename(dirname($file)); + my $show = basename(dirname(dirname($file))); + #print "-> Found show $show season $season and recording $recording\n"; + $tv_show_dirs{$show}{$season}{has_recordings} = 1; + $tv_show_dirs{$show}{has_recordings} = 1; +} + +#print Dumper(\%tv_show_dirs); + +# For each season directory that doesn't have recordings, remove it +foreach $show (keys %tv_show_dirs) { + foreach $season (keys %{$tv_show_dirs{$show}}) { + if($season eq "has_recordings") { next; } + if ($tv_show_dirs{$show}{$season}{has_recordings} == 0) { + print "-> Removing season dir '$show/$season'\n"; + system("rm -rf \"$tv_src_dir/$show/$season/\""); + } + } +} + +# For each show directory that doesn't have any recordings under it, remove it +foreach $show (keys %tv_show_dirs) { + my $has_recordings = 0; + if($show eq ".grab") { next; } + foreach $season (keys %{$tv_show_dirs{$show}}) { + if ($tv_show_dirs{$show}{$season}{has_recordings} == 1) { + $has_recordings = 1; + } + } + if($has_recordings == 0) { + print "-> Removing show dir '$show'\n"; + system("rm -rf \"$tv_src_dir/$show/\""); + } +}