Merge 'Tree file management strategy and file backups'

Reviewed-on: #3

CHANGELOG

@@ -1,9 +1,17 @@
-0.4.1 2024-07-14: Fixes "Cancel" option not exiting on package resolution
-                  Fixes whitespace preventing package list matches on "remove all"
+0.6.0 2024-09-03: File management with the tree strategy
+									File backups
+                  Extract authorization command to top level
+                  ask and tildify utility functions
+0.5.0 2024-07-18: "Decide in editor" package conflict resolution strategy
+                  Drop pipefail shell option for dash compatibility
+                  Add numerical debug levels to the log utility function
+                  Check for shell option support before setting
+0.4.1 2024-07-14: Fix "Cancel" option not exiting on package resolution
+                  Fix whitespace preventing package list matches on "remove all"
 0.4.0 2024-07-14: "Add/remove all from configuration" resolution strategy
 0.3.1 2024-07-13: Refactor, new docs page, make cache refresh lazier
 0.3.0 2024-07-11: "Enter packages to install/uninstall" resolution strategy
 0.2.1 2024-07-10: Rename package_resolution.sh, document package conflicts
 0.2.0 2024-07-10: Implement "uninstall/instal all" package resolution strategies
 0.1.1 2024-07-07: Handle missing tori.conf and invalid installation root path
-0.1.0 2024-07-07: Added configuration parsing
+0.1.0 2024-07-07: Add configuration parsing

README.md

@@ -2,7 +2,7 @@
 
 tori is a tool to track your personal systems' configurations and replicate them.
 
-If you'd like a more detailed description of what it is, its purpose, origins and goals, see the [announcement blog post](https://blog.jutty.dev/posts/introducing-tori.html).
+If you'd like a more detailed description of what it is, its purpose, origins and goals, see the [announcement blog post](https://blog.jutty.dev/posts/introducing-tori/).
 
 Refer to the [project website](https://tori.jutty.dev) for updates and access to [documentation](https://tori.jutty.dev/docs).
 

src/check.sh

@@ -6,4 +6,5 @@ check() {
   log debug "collected bkp files:\n$bkp_files"
 
   scan_packages
+  merge_files "$base_files"
 }

src/configuration.sh

@@ -15,7 +15,7 @@ scan_directory() {
     done
   fi
 
-  echo "$files"
+  printf "%b" "$files"
 }
 
 scan_packages() {
@@ -29,12 +29,8 @@ scan_packages() {
   user_packages="$(get_user_packages)"
 
   if [ "$system_packages" = "$user_packages" ]; then
-    log debug "packages match"
+    log debug "Packages match"
   else
-    log debug "packages mismatch"
-    log debug "system:\n$system_packages"
-    log debug "user:\n$user_packages"
-
     log user "System and configuration packages differ"
     resolve_packages
   fi

src/file/backup.sh

@@ -0,0 +1,40 @@
+# takes a list of newline-separated absolute paths 
+# backs each path up, creating canonical or ephemeral copies as needed
+backup_paths() {
+  local paths="$1"
+  local canonical_path=
+  local ephemeral_path=
+
+  for path in $paths; do
+    canonical_path="$BACKUP_ROOT/canonical$path"
+    timestamp="$(date +'%Y-%m-%dT%H-%M-%S')"
+    ephemeral_path="$BACKUP_ROOT/ephemeral${path}_$timestamp"
+
+    if [ -f "$canonical_path" ]; then
+      log debug "[backup] Creating ephemeral copy for $path"
+      mkdir -p "$(dirname "$ephemeral_path")"
+      if [ -f "$ephemeral_path" ]; then
+        log debug "[backup] Overwriting ephemeral copy for $path"
+        if [ -r "$path" ]; then
+          cp -f "$path" "$ephemeral_path"
+        else
+          $AUTHORIZE_COMMAND cp -f "$path" "$ephemeral_path"
+        fi
+      else
+        if [ -r "$path" ]; then
+          cp "$path" "$ephemeral_path"
+        else
+          $AUTHORIZE_COMMAND cp "$path" "$ephemeral_path"
+        fi
+      fi
+    else
+      log debug "[backup] Creating canonical copy for $path"
+      mkdir -p "$(dirname "$canonical_path")"
+      if [ -r "$path" ]; then
+        cp "$path" "$canonical_path"
+      else
+        $AUTHORIZE_COMMAND cp "$path" "$canonical_path"
+      fi
+    fi
+  done
+}

src/file/file_merge.sh

@@ -0,0 +1,85 @@
+merge_files() {
+  local base_files="$1"
+  local strategy="${2:-tree}"
+
+  if [ "$strategy" == tree ]; then
+    log info "[merge_files] Merging with $strategy strategy"
+    if ! file_scan_tree "$base_files"; then
+      if ! file_merge_tree "$base_files"; then
+        merge_files "$base_files"
+      fi
+    fi
+  fi
+}
+
+file_scan_tree() {
+  local base_files="$1"
+
+  for file in $base_files; do
+    local absolute_path="$(echo "$file" | sed 's/^base//')"
+    local config_path="$CONFIG_ROOT/$file"
+
+    if ! diff "$absolute_path" "$config_path" > /dev/null 2>&1; then
+      return 1
+    fi
+  done
+  return 0
+}
+
+file_merge_tree() {
+  local base_files="$1"
+  local overwrite_choice=
+
+  for file in $base_files; do
+    log debug "[merge_tree] Processing $file"
+    local absolute_path="$(echo "$file" | sed 's/^base//')"
+    log debug "[merge_tree] Absolute path: $absolute_path"
+    local config_path="$CONFIG_ROOT/$file"
+    log debug "[merge_tree] Config path: $config_path"
+
+    if diff "$absolute_path" "$config_path" > /dev/null 2>&1; then
+      log debug "[merge_tree] Files match"
+    else
+      log debug "[merge_tree] Files differ"
+      local prompt_verb="Differs"
+        local prompt_options="Overwrite system,Overwrite configuration,Show difference"
+      if ! [ -f "$absolute_path" ]; then
+        local prompt_verb="In configuration only"
+        local prompt_options="Copy to system"
+      fi
+      overwrite_choice="$(ask "$prompt_verb: $(tildify "$absolute_path")" "$prompt_options")"
+      log debug "[merge_tree] Overwrite choice: $overwrite_choice"
+
+      if [ "$overwrite_choice" -eq 0 ]; then
+        return 0
+      elif [ "$overwrite_choice" -eq 1 ]; then
+        backup_paths "$absolute_path"
+        if [ -r "$config_path" ] && [ -w "$absolute_path" ]; then
+          cp -vi "$config_path" "$absolute_path"
+        else
+          $AUTHORIZE_COMMAND cp -vi "$config_path" "$absolute_path"
+        fi
+        return 1
+      elif [ "$overwrite_choice" -eq 2 ]; then
+        backup_paths "$config_path"
+        if [ -r "$absolute_path" ] && [ -w "$config_path" ]; then
+          cp -vi "$absolute_path" "$config_path"
+        else
+          $AUTHORIZE_COMMAND cp -vi "$absolute_path" "$config_path"
+        fi
+        return 1
+      elif [ "$overwrite_choice" -eq 3 ]; then
+        echo "< $(tildify "$absolute_path") | $(echo "$config_path" | sed "s*$CONFIG_ROOT/**") >"
+        if [ -r "$absolute_path" ] && [ -r "$config_path" ]; then
+          diff "$absolute_path" "$config_path"
+        else
+          $AUTHORIZE_COMMAND diff "$absolute_path" "$config_path"
+        fi
+        return 1
+      else
+        log user 'Invalid choice'
+        return 1
+      fi
+    fi
+  done
+}

src/index.sh

@@ -7,4 +7,8 @@
 . "$TORI_ROOT/src/package/package_conflict_resolution.sh"
 . "$TORI_ROOT/src/package/package_tracking.sh"
 . "$TORI_ROOT/src/package/validate_input_packages.sh"
+. "$TORI_ROOT/src/package/package_conflict_input_parser.sh"
 . "$TORI_ROOT/src/package/update_package_cache.sh"
+
+. "$TORI_ROOT/src/file/file_merge.sh"
+. "$TORI_ROOT/src/file/backup.sh"

src/package/package_conflict_input_parser.sh

@@ -0,0 +1,80 @@
+package_conflict_input_parser() {
+  local packages="$1"
+  local conflict_type="$2"
+  local input="$TMP_ROOT/package_conflict_input"
+  local input_choices="$TMP_ROOT/package_conflict_input_choices"
+  local choices=
+  local packages_to_install=
+  local packages_to_uninstall=
+  local packages_to_track=
+  local packages_to_untrack=
+
+    help_text_generator "$conflict_type" > "$input"
+
+  echo "$packages" | sed 's/ /\n/g' | while read -r package; do
+    echo "skip $package" >> "$input"
+  done
+
+  $EDITOR "$input"
+
+  choices="$(cat "$input" | grep -v '^#' | grep '.')"
+  echo "$choices" > "$input_choices"
+
+  # validation
+  while read -r action package; do
+
+    validate_input_packages "$package"
+
+    if [ "$action" = install ] || [ "$action" = i ]; then
+      packages_to_install="$packages_to_install $package"
+    elif [ "$action" = uninstall ] || [ "$action" = u ]; then
+      packages_to_uninstall="$packages_to_uninstall $package"
+    elif [ "$action" = add ] || [ "$action" = a ]; then
+      packages_to_track="$packages_to_track $package"
+    elif [ "$action" = remove ] || [ "$action" = r ]; then
+      packages_to_untrack="$packages_to_untrack $package"
+    elif [ "$action" = skip ] || [ "$action" = s ]; then
+      log debug "[package_conflict_input_parser] Skipped: $package"
+    else
+      log user "Invalid action provided for $package: $action"
+    fi
+  done < "$input_choices"
+
+  # actual system or configuration change
+  if [ -n "$packages_to_install" ]; then
+    package_manager install "$packages_to_install"
+  fi
+
+  if [ -n "$packages_to_uninstall" ]; then
+    package_manager uninstall "$packages_to_uninstall"
+  fi
+
+  if [ -n "$packages_to_track" ]; then
+    track_packages "$packages_to_track"
+  fi
+
+  if [ -n "$packages_to_untrack" ]; then
+    untrack_packages "$packages_to_untrack"
+  fi
+}
+
+help_text_generator() {
+  local conflict_type="$1"
+
+  echo "# Options:"
+
+  if [ "$conflict_type" == not_installed ]; then
+    echo "#   [i]nstall     Install package to system"
+    echo "#   [r]emove      Remove from configuration"
+  elif [ "$conflict_type" == not_on_configuration ]; then
+    echo "#   [u]ninstall   Uninstall package from system"
+    echo "#   [a]dd         Add to configuration"
+  else
+    debug fatal "Invalid conflict type provided: $conflict_type"
+    return 1
+  fi
+
+  echo "#   [s]kip        Do not take any action"
+  echo -e "\n# Providing just the value between brackets is sufficient"
+  echo -e "# Replace 'skip' below with the desired option\n"
+}

src/package/package_conflict_resolution.sh

@@ -3,18 +3,18 @@ resolve_packages() {
   local input_packages=
 
   # shellcheck disable=SC2154
-  ( echo "$system_packages" > "$TMP_DIR/system_packages"
-    echo "$user_packages" > "$TMP_DIR/user_packages" )
+  ( echo "$system_packages" > "$TMP_ROOT/system_packages"
+    echo "$user_packages" > "$TMP_ROOT/user_packages" )
 
   local packages_not_on_configuration="$(grep -v -x -f \
-    "$TMP_DIR/user_packages" "$TMP_DIR/system_packages" | xargs)"
+    "$TMP_ROOT/user_packages" "$TMP_ROOT/system_packages" | xargs)"
 
   if [ -n "$packages_not_on_configuration" ]; then
     not_on_configuration_dialog "$packages_not_on_configuration"
   fi
 
   local packages_not_installed=$(grep -v -x -f \
-    "$TMP_DIR/system_packages" "$TMP_DIR/user_packages" | xargs)
+    "$TMP_ROOT/system_packages" "$TMP_ROOT/user_packages" | xargs)
 
   if [ -n "$packages_not_installed" ]; then
     not_installed_dialog "$packages_not_installed"
@@ -57,6 +57,8 @@ not_on_configuration_dialog() {
     if validate_input_packages "$input_packages"; then
       track_packages "$input_packages"
     fi
+  elif [ "$strategy" = 5 ]; then
+    package_conflict_input_parser "$conflicted_packages" 'not_on_configuration'
   else
     log debug "[resolve_packages] Unexpected input: $strategy"
     not_on_configuration_dialog "$conflicted_packages"
@@ -97,6 +99,8 @@ not_installed_dialog() {
     read -r -p "Enter space-separated packages to remove from the configuation: " input_packages
     log debug "Input: input_packages = $input_packages"
     untrack_packages "$input_packages"
+  elif [ "$strategy" = 5 ]; then
+    package_conflict_input_parser "$conflicted_packages" 'not_installed'
   else
     log debug "[resolve_packages] Unexpected input: $strategy"
     not_installed_dialog "$conflicted_packages"

src/package/package_manager.sh

@@ -2,15 +2,14 @@ package_manager() {
   local command="$1"
 
   local manager
-  local authorizer="sudo"
   local args__install
   local args__uninstall
   local args__get_manually_installed
   local args__get_available
 
-  set_opts +
+  set_opts off
   local args__user_args="$2"
-  set_opts -
+  set_opts on
 
   if [ "$OS" = "FreeBSD" ]; then
     manager="pkg"
@@ -25,11 +24,11 @@ package_manager() {
   if [ "$command" = 'get_manually_installed' ]; then
     eval $manager "$args__get_manually_installed"
   elif [ "$command" = 'install' ]; then
-    $authorizer $manager $args__install $args__user_args
+    $AUTHORIZE_COMMAND $manager $args__install $args__user_args
   elif [ "$command" = 'uninstall' ]; then
-    $authorizer $manager $args__uninstall $args__user_args
+    $AUTHORIZE_COMMAND $manager $args__uninstall $args__user_args
   elif [ "$command" = 'update' ]; then
-    $authorizer $manager $args__update
+    $AUTHORIZE_COMMAND $manager $args__update
   elif [ "$command" = 'get_available' ]; then
     eval $manager "$args__get_available"
   else

src/package/package_tracking.sh

@@ -9,10 +9,10 @@ track_packages() {
 untrack_packages() {
   local packages="$1"
 
-  log debug "[untrack_packages] Removing packages: $packages"
+  log info "[untrack_packages] Removing packages: $packages"
 
   echo "$packages" | xargs | sed 's/ /\n/g' | while read -r package; do
     sed -i '' "/^[[:space:]]*$package[[:space:]]*$/d" "$CONFIG_ROOT/packages"
-    log debug "[untrack_packages] Executed removal for $package with exit code $?"
+    log info "[untrack_packages] Executed removal for $package with exit code $?"
   done
 }

src/package/update_package_cache.sh

@@ -1,7 +1,7 @@
 update_package_cache() {
-  set_opts +
+  set_opts off
   local argument="$1"
-  set_opts -
+  set_opts on
 
   if [ -f "$PACKAGE_CACHE" ]; then
     local last_update="$(date -r "$PACKAGE_CACHE" +%Y-%m-%d)"

src/utility.sh

@@ -4,30 +4,142 @@ log() {
   local level="$1"
   local message="$2"
 
-  if [ $level = fatal ]; then
-    printf "[tori] $(date "+%H:%M:%S"): $message\n" 1>&2
-  elif [ $level = user ]; then
-    printf "[tori] $(date "+%H:%M:%S"): $message\n" 1>&2
-  elif [ -n "$DEBUG" ] && [ $level = debug ]; then
-    printf "$(date "+%H:%M:%N") $message\n" 1>&2
+  print_user_message() {
+    printf "%b\n" "[tori] $(date "+%H:%M:%S"): $1" 1>&2
+  }
+
+  print_debug_message() {
+    printf "%b\n" "$(date "+%H:%M:%N") $1" 1>&2
+  }
+
+  if [ -z "$DEBUG" ]; then
+    DEBUG=3
+  elif ! echo "$DEBUG" | grep -q '^[[:number:]]$'; then
+    echo "[log] Warning: DEBUG should always be set to a number. Assuming DEBUG=3 (warn)"
+    DEBUG=3
   fi
+
+  if [ -z "$DEBUG_DISABLED_WARNING" ] && [ "$DEBUG" -eq 0 ]; then
+    echo "[log] Warning: Setting DEBUG=0 disables all logging except for user messages"
+    echo "      Use a value beween 1 (fatal) and 5 (debug). The default level is 3 (warn)"
+    DEBUG_DISABLED_WARNING=1
+  elif [ "$DEBUG" -gt 5 ]; then
+    echo "[log] Warning: Assuming DEBUG maximum level of 5 (debug) over provided level $DEBUG"
+    DEBUG=5
+  fi
+
+  if [ "$level" = user ]; then
+    print_user_message "$message"
+  elif [ "$DEBUG" -ge 1 ] && [ "$level" = fatal ]; then
+    print_user_message "$message"
+  elif [ "$DEBUG" -ge 2 ] && [ "$level" = error ]; then
+    print_user_message "$message"
+  elif [  "$DEBUG" -ge 3 ] && [ "$level" = warn ]; then
+    print_user_message "$message"
+  elif [ "$DEBUG" -ge 4 ] && [ "$level" = info ]; then
+    print_debug_message "$message"
+  elif [ "$DEBUG" -ge 5 ] && [ "$level" = debug ]; then
+    print_debug_message "$message"
+  fi
+}
+
+confirm() {
+  local question="$1"
+  local answer=
+  read -rp "$question [y/N] " answer
+
+  if [ "$answer" == y ] || [ "$answer" == Y ]; then
+    return 0;
+  else
+    return 1;
+  fi
+}
+
+ask() {
+  local question="$1"
+  local options="$2"
+  local answer=
+  local options_count=0
+  local dialog_options=
+
+  local IFS=,
+  for option in $options; do
+    _=$((options_count+=1))
+    dialog_options="$dialog_options\n [$options_count] $option"
+  done;
+  IFS=
+  dialog_options="$dialog_options\n [0] Exit"
+
+  printf "%s" "$question" >&2
+  printf "%b" "$dialog_options" >&2
+  printf "\n%s" "Choose an option number: " >&2
+  read -r read_answer
+  answer="$(echo "$read_answer" | xargs)"
+
+  if [ -z "$answer" ]; then
+    log info "[ask] Invalid choice"
+    echo -1
+    return 1
+  elif [ "$answer" -ge 0 ] 2> /dev/null && [ "$answer" -le $options_count ]; then
+    echo "$answer"
+  else
+    log info "[ask] Invalid choice"
+    echo -1
+    return 1
+  fi
+}
+
+tildify() {
+  echo "$1" | sed "s*$HOME*~*"
 }
 
 set_opts() {
-  sign="$1"
+  local target="$1"
+  local sign=
 
-  set "${sign}o" errexit
-  set "${sign}o" nounset
-  set "${sign}o" pipefail
+  if [ "$target" = on ]; then
+    sign='-'
+  elif [ "$target" = off ]; then
+    sign='+'
+  else
+    log fatal "Invalid set_opts target: $target. Expected on or off"
+    return 1
+  fi
+
+  set_opt() {
+    local opt="$1"
+
+    if set -o | grep -q "^$opt[[:space:]]"; then
+      set "${sign}o" "$opt"
+      log debug "[set_opts] Set: $(set -o | grep "^$opt[[:space:]]")"
+    else
+      log fatal "Unsupported shell: no $opt option support"
+      return 1
+    fi
+  }
+
+  set_opt errexit
+  set_opt nounset
+  set_opt pipefail
 }
 
 prepare_directories() {
-  if ! [ -d "$TMP_DIR" ]; then
-    mkdir "$TMP_DIR"
+  if ! [ -d "$TMP_ROOT" ]; then
+    mkdir "$TMP_ROOT"
   fi
 
-  if ! [ -d "$CACHE_DIR" ]; then
-    mkdir -p "$CACHE_DIR"
+  if ! [ -d "$CACHE_ROOT" ]; then
+    mkdir -p "$CACHE_ROOT"
+  fi
+
+  if ! [ -d "$BACKUP_ROOT" ]; then
+    mkdir -p "$BACKUP_ROOT"
+    if ! [ -d "$BACKUP_ROOT/canonical" ]; then
+      mkdir "$BACKUP_ROOT/canonical"
+    fi
+    if ! [ -d "$BACKUP_ROOT/ephemeral" ]; then
+      mkdir "$BACKUP_ROOT/ephemeral"
+    fi
   fi
 
   if ! [ -d "$CONFIG_ROOT" ]; then

tori

@@ -2,31 +2,32 @@
 
 main() {
   # paths
-  VERSION="0.4.1 2024-07-14"
+  VERSION="0.6.0 2024-09-03"
   TORI_ROOT="$HOME/.local/share/tori"
   CONFIG_ROOT="$HOME/.config/tori"
-  TMP_DIR="/tmp/tori"
-  CACHE_DIR="$HOME/.cache/tori"
+  BACKUP_ROOT="$HOME/.local/state/tori/backup"
+  TMP_ROOT="/tmp/tori"
+  CACHE_ROOT="$HOME/.cache/tori"
 
-  check_core_paths
-
-  . "$TORI_ROOT/src/index.sh"
-
-  # state
+  # os-independent state
 
   DEBUG=$DEBUG
+  DEBUG_DISABLED_WARNING=
 
   ## user input
   argument="$1"
   parameter="$2"
 
-  set_opts -
+  # import source
+  check_core_paths
+  . "$TORI_ROOT/src/index.sh"
 
-  ## global constants
+  ## os-dependent state
+  set_opts on
   OS="$(get_operating_system)"
-  PACKAGE_CACHE="$CACHE_DIR/${OS}_packages.cache"
+  PACKAGE_CACHE="$CACHE_ROOT/${OS}_packages.cache"
+  AUTHORIZE_COMMAND="sudo"
 
-  ## global state
   base_files=
   bkp_files=
   user_packages=
@@ -69,7 +70,6 @@ check_core_paths() {
     exit 1
   fi
 
-  [ -n "$DEBUG" ] && echo "TORI_ROOT: $TORI_ROOT"
 }
 
 main "$1" "$2"