Added the downsync utility (both Bash and Powershell) to retrieve exported backups.
This commit is contained in:
233
.utils/downsync/downsync.ps1
Normal file
233
.utils/downsync/downsync.ps1
Normal file
@@ -0,0 +1,233 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Powershell script for one-way (down) synchronization of a remote web folder.
|
||||
|
||||
.DESCRIPTION
|
||||
It does not handle any remote subfolders, only the root folder. Downloads
|
||||
all files that do not exist locally. Updates only an existing file that is
|
||||
older than the remote source. It warns of errors or possible inconsistencies.
|
||||
|
||||
Creates a unique log file in the local folder (this can be disabled).
|
||||
|
||||
Usage: $PSCommandPath -Remote URI_to_be_synced [-Local local_folder]
|
||||
[-User username] [-Pass base64_encoded_password]
|
||||
[-NoLog] [-Info]
|
||||
|
||||
Author: Zolt<6C>n KOV<4F>CS <kovacsz@marcusconsulting.hu>
|
||||
License: GNU/GPL 3+ https://www.gnu.org/licenses/gpl-3.0.html
|
||||
|
||||
.NOTES
|
||||
Changelog:
|
||||
2025-03-12 v0.1 Initial release.
|
||||
#>
|
||||
|
||||
# Command line parameters.
|
||||
#
|
||||
param (
|
||||
# An http(s) URI pointing to a remote web folder containing the files to synchronize.
|
||||
[Parameter()][string]$Remote,
|
||||
# An existing and writable local folder where the script will download the files.
|
||||
[Parameter()][string]$Local = $PSScriptRoot,
|
||||
# Credentials, if required by the remote website.
|
||||
[Parameter()][string]$User,
|
||||
# A base64-encoded password (if necessary).
|
||||
[Parameter()][string]$Pass,
|
||||
# On error the script will try to download a file at most this many times. Defaults to 3 tries.
|
||||
[Parameter()][int]$MaxTries,
|
||||
# On error the script will wait this many seconds between two download attempts. Defaults to 5 seconds.
|
||||
[Parameter()][int]$WaitRetry,
|
||||
# The script warns if the downloaded file is shorter than this value. Defaults to 1024 bytes.
|
||||
[Parameter()][int]$SmallSize,
|
||||
# If set, the script will not write log file.
|
||||
[Parameter()][switch]$NoLog = $false,
|
||||
# If set, the script will display log lines.
|
||||
[Parameter()][switch]$Info = $false
|
||||
)
|
||||
|
||||
# Initialisations.
|
||||
#
|
||||
if (-not $MaxTries) { $MaxTries = 3 }
|
||||
if (-not $SmallSize) { $SmallSize = 1024 }
|
||||
if (-not $WaitRetry) { $WaitRetry = 5 }
|
||||
|
||||
# Messages.
|
||||
#
|
||||
$Message = @{}
|
||||
$Message['Bad DNS'] = 'Remote host is not an IP and not resolvable.'
|
||||
$Message['Bad folder'] = "The local path must point to a writable folder."
|
||||
$Message['Bad URI'] = 'Remote parameter must be a valid http(s) URI.'
|
||||
$Message['Collision array'] = "The local path is an existing array:"
|
||||
$Message['Downloaded'] = "Downloaded file:"
|
||||
$Message['Empty filelist'] = "List of files is empty:"
|
||||
$Message['Finished'] = "Synchronisation finished."
|
||||
$Message['Is a folder'] = "Remote subfolders are ignored:"
|
||||
$Message['Local newer'] = "The files are different but the local one is newer:"
|
||||
$Message['Size mismatch'] = "Size of the downloaded file differ:"
|
||||
$Message['Started'] = "Sychronisation started."
|
||||
$Message['Unable fetchdir'] = "Unable to fetch the content of the remote folder."
|
||||
$Message['Unable to decode'] = 'Password must be properly base64 encoded.'
|
||||
$Message['Unable to stat remote'] = 'Unable to stat the remote object:'
|
||||
$Message['Small file'] = "File is smaller than " + $SmallSize + " bytes:"
|
||||
|
||||
$Message['Usage'] = "Usage:`n" + `
|
||||
$PSCommandPath + ' -Remote URI_to_be_synced [-Local local_folder] ' + `
|
||||
'[-User username] [-Pass base64_encoded_password] ' + `
|
||||
'[-NoLog True] [-Info True]'
|
||||
|
||||
# Logger function.
|
||||
#
|
||||
function Write-Log {
|
||||
$date = Get-Date -Format "yyyy-MM-dd HH:mm:ss.fff"
|
||||
if ( -not($NoLog)) { Add-Content -Path $LogFilePath -Value "$date $args" }
|
||||
if ($Info) { Write-Host $args }
|
||||
}
|
||||
|
||||
# Checks the -Remote parameter.
|
||||
#
|
||||
# It is mandatory.
|
||||
if ( -not("$Remote")) { Write-Host $Message['Usage']; exit 1 }
|
||||
# The closing / is necessary.
|
||||
$Remote = $Remote.TrimEnd('/') + '/'
|
||||
# Must be well-formed and http(s).
|
||||
if ( -not([uri]::IsWellFormedUriString("$Remote", 'Absolute')) -or -not(([uri] "$Remote").Scheme -in 'http', 'https')) {
|
||||
Write-Host $Message['Bad URI']; exit 1 }
|
||||
# Must be IPv4 or resolvable.
|
||||
if ( -not(([uri]"$Remote").Host -match "^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$" -and [bool](([uri]"$Remote").Host -as [ipaddress]))) {
|
||||
# It is resolvable?
|
||||
try { Resolve-DnsName -Name ([uri]"$Remote").Host -ErrorAction Stop | Out-Null }
|
||||
catch { Write-Host $Message['Bad DNS']; exit 1 }
|
||||
}
|
||||
#
|
||||
# We've a somewhat checked remote.
|
||||
|
||||
# Checks the -Local parameter.
|
||||
#
|
||||
# Must be an existing, writable folder.
|
||||
if ( -not("$Local")) { $Local = $PSScriptRoot }
|
||||
if ( -not(Test-Path -LiteralPath "$Local" -pathType Container)) { Write-Host $Message['Bad folder']; exit 1 }
|
||||
# Can we write into?
|
||||
try {
|
||||
$testfile = $Local + '\' + [guid]::NewGuid() + '.tmp'
|
||||
[io.file]::OpenWrite("$testfile").close()
|
||||
Remove-Item -ErrorAction SilentlyContinue "$testfile" }
|
||||
catch { Write-Host $Message['Bad folder']; exit 1 }
|
||||
#
|
||||
# We've a somewhat checked local folder.
|
||||
|
||||
# Decodes the provided -Pass (if any).
|
||||
#
|
||||
if ("$Pass") {
|
||||
try { $Pass = ([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($Pass))) }
|
||||
catch { Write-Host $Message['Unable to decode']; exit 1 }
|
||||
}
|
||||
#
|
||||
# We've a decoded (or empty) password.
|
||||
|
||||
# Initializes the log file.
|
||||
#
|
||||
$LogFilePath = $Local + '\' + (Get-Item $PSCommandPath ).Basename + (Get-Date -Format "-yyyyMMdd-HHmmss") +'.log'
|
||||
Write-Log $Message['Started']
|
||||
#
|
||||
# We've the log file ready to use.
|
||||
|
||||
# Prepares the Authorization header from provided credentials (if any).
|
||||
#
|
||||
$Headers = ''
|
||||
if ("$User" ) {
|
||||
$encoded = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes("$($User):$($Pass)"))
|
||||
$Headers = @{ Authorization = "Basic $encoded" }
|
||||
}
|
||||
# We've an Authorization header ready to use for Webrequests.
|
||||
|
||||
# Let's get the directory index from the remote source.
|
||||
#
|
||||
$response = ''
|
||||
try {
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
if ("$Headers") {$response = (Invoke-WebRequest -Uri "$Remote" -Headers $Headers -UseBasicParsing) }
|
||||
else {$response = (Invoke-WebRequest -Uri "$Remote" -UseBasicParsing ) }
|
||||
}
|
||||
catch { Write-Log $Message['Unable fetchdir'] "$Remote" $_.Exception.Response.StatusCode.Value__ $_.Exception.Response.StatusDescription; exit 1 }
|
||||
$files = @($response.Links.HREF | select -skip 1)
|
||||
#
|
||||
# We send a warning if it is empty.
|
||||
#
|
||||
if ($files.Count -eq 0) { Write-Log $Message['Empty filelist'] "$Remote" }
|
||||
#
|
||||
# We've the list of remote files.
|
||||
|
||||
# Processes the remote files in a row, one after the other.
|
||||
#
|
||||
foreach ($file in $files) {
|
||||
#
|
||||
# Let's get the parameters of the remote object. On error we send a warning and move on.
|
||||
#
|
||||
$remoteHeaders = ''
|
||||
try {
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
if ("$Headers") { $remoteHeaders = (Invoke-WebRequest -Uri ("$Remote" + "$file") -Headers $Headers -Method Head -UseBasicParsing ).Headers }
|
||||
else { $remoteHeaders = (Invoke-WebRequest -Uri ("$Remote" + "$file") -Method Head -UseBasicParsing).Headers }
|
||||
}
|
||||
catch { Write-Log $Message['Unable to stat remote'] ("$Remote" + "$file") $_.Exception.Message; continue }
|
||||
$remoteDate = $remoteHeaders['Last-Modified']
|
||||
$remoteSize = $remoteHeaders['Content-Length']
|
||||
$remoteType = $remoteHeaders['Content-Type']
|
||||
#
|
||||
# If the remote object is a folder we send a warning and move on.
|
||||
#
|
||||
if ("$remoteType" -eq 'text/directory') { Write-Log $Message['Is a folder'] ("$Remote" + "$file"); continue }
|
||||
#
|
||||
# If we've a local object and it is a folder we send a warning and move on.
|
||||
#
|
||||
if (Test-Path -LiteralPath "$Local\$file" -PathType Container) { Write-Log $Message['Collision array'] "$Local\$file"; continue }
|
||||
#
|
||||
# We've an existing local file?
|
||||
#
|
||||
if (Test-Path -LiteralPath "$Local\$file" -PathType Leaf) {
|
||||
$localDate = (Get-Item -LiteralPath ("$Local" + '\' + "$file")).LastWriteTime.DateTime
|
||||
$localSize = (Get-Item -LiteralPath ("$Local" + '\' + "$file")).Length
|
||||
#
|
||||
# If the local file is newer than remote we don't replace it, but we send a warning if the sizes are different.
|
||||
#
|
||||
if ((Get-Date $localDate) -gt (Get-Date $remoteDate)) {
|
||||
if ( $localSize -ne $remoteSize ) { Write-Log $Message['Local newer'] $file }
|
||||
continue
|
||||
}
|
||||
}
|
||||
#
|
||||
# OK, we decided to download the remote file.
|
||||
# On failure, we'll try again a few times.
|
||||
#
|
||||
for ($i = 1; $i -le $MaxTries; $i++) {
|
||||
try {
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
if ("$Headers") { Invoke-WebRequest -Uri ("$Remote" + "$file") -Headers $Headers -OutFile ($Local + '\' + $file) }
|
||||
else { Invoke-WebRequest -Uri ("$Remote" + "$file") -OutFile ($Local + '\' + $file) }
|
||||
#
|
||||
Write-Log $Message['Downloaded'] ("$Remote" + "$file")
|
||||
#
|
||||
# Checks the size of the downloaded file, stops trying if it is OK.
|
||||
#
|
||||
$localSize = (Get-Item -LiteralPath ("$Local" + '\' + "$file")).Length
|
||||
if ( $localSize -eq $remoteSize ) {
|
||||
#
|
||||
# We send a warning on small files (except the logs).
|
||||
#
|
||||
if ($localSize -lt $SmallSize -and (Get-Item ("$Local" + "\" + "$file")).Extension -notin ('.log')) {
|
||||
Write-Log $Message['Small file'] ("$Local" + "\" + "$file") }
|
||||
break
|
||||
}
|
||||
#
|
||||
Write-Log $Message['Size mismatch'] $Local\$file $localSize $remoteSize
|
||||
}
|
||||
catch { Write-Log $Message['Unable to download'] ("$Remote" + "$file") $_.Exception.Message }
|
||||
#
|
||||
# Waits before retrying.
|
||||
#
|
||||
Start-Sleep -Seconds $WaitRetry
|
||||
}
|
||||
}
|
||||
#
|
||||
# That's all.
|
||||
#
|
||||
Write-Log $Message['Finished']
|
||||
Reference in New Issue
Block a user