234 lines
9.6 KiB
PowerShell
234 lines
9.6 KiB
PowerShell
<#
|
|
.SYNOPSIS
|
|
Powershell script for one-way (down) synchronization of a remote web folder.
|
|
|
|
.DESCRIPTION
|
|
It does not handle any remote subfolders, only the root folder. Downloads
|
|
all files that do not exist locally. Updates only an existing file that is
|
|
older than the remote source. It warns of errors or possible inconsistencies.
|
|
|
|
Creates a unique log file in the local folder (this can be disabled).
|
|
|
|
Usage: $PSCommandPath -Remote URI_to_be_synced [-Local local_folder]
|
|
[-User username] [-Pass base64_encoded_password]
|
|
[-NoLog] [-Info]
|
|
|
|
Author: Zoltán KOVÁCS <kovacsz@marcusconsulting.hu>
|
|
License: GNU/GPL 3+ https://www.gnu.org/licenses/gpl-3.0.html
|
|
|
|
.NOTES
|
|
Changelog:
|
|
2025-03-12 v0.1 Initial release.
|
|
#>
|
|
|
|
# Command line parameters.
|
|
#
|
|
param (
|
|
# An http(s) URI pointing to a remote web folder containing the files to synchronize.
|
|
[Parameter()][string]$Remote,
|
|
# An existing and writable local folder where the script will download the files.
|
|
[Parameter()][string]$Local = $PSScriptRoot,
|
|
# Credentials, if required by the remote website.
|
|
[Parameter()][string]$User,
|
|
# A base64-encoded password (if necessary).
|
|
[Parameter()][string]$Pass,
|
|
# On error the script will try to download a file at most this many times. Defaults to 3 tries.
|
|
[Parameter()][int]$MaxTries,
|
|
# On error the script will wait this many seconds between two download attempts. Defaults to 5 seconds.
|
|
[Parameter()][int]$WaitRetry,
|
|
# The script warns if the downloaded file is shorter than this value. Defaults to 1024 bytes.
|
|
[Parameter()][int]$SmallSize,
|
|
# If set, the script will not write log file.
|
|
[Parameter()][switch]$NoLog = $false,
|
|
# If set, the script will display log lines.
|
|
[Parameter()][switch]$Info = $false
|
|
)
|
|
|
|
# Initialisations.
|
|
#
|
|
if (-not $MaxTries) { $MaxTries = 3 }
|
|
if (-not $SmallSize) { $SmallSize = 1024 }
|
|
if (-not $WaitRetry) { $WaitRetry = 5 }
|
|
|
|
# Messages.
|
|
#
|
|
$Message = @{}
|
|
$Message['Bad DNS'] = 'Remote host is not an IP and not resolvable.'
|
|
$Message['Bad folder'] = "The local path must point to a writable folder."
|
|
$Message['Bad URI'] = 'Remote parameter must be a valid http(s) URI.'
|
|
$Message['Collision array'] = "The local path is an existing array:"
|
|
$Message['Downloaded'] = "Downloaded file:"
|
|
$Message['Empty filelist'] = "List of files is empty:"
|
|
$Message['Finished'] = "Synchronisation finished."
|
|
$Message['Is a folder'] = "Remote subfolders are ignored:"
|
|
$Message['Local newer'] = "The files are different but the local one is newer:"
|
|
$Message['Size mismatch'] = "Size of the downloaded file differ:"
|
|
$Message['Started'] = "Sychronisation started."
|
|
$Message['Unable fetchdir'] = "Unable to fetch the content of the remote folder."
|
|
$Message['Unable to decode'] = 'Password must be properly base64 encoded.'
|
|
$Message['Unable to stat remote'] = 'Unable to stat the remote object:'
|
|
$Message['Small file'] = "File is smaller than " + $SmallSize + " bytes:"
|
|
|
|
$Message['Usage'] = "Usage:`n" + `
|
|
$PSCommandPath + ' -Remote URI_to_be_synced [-Local local_folder] ' + `
|
|
'[-User username] [-Pass base64_encoded_password] ' + `
|
|
'[-NoLog True] [-Info True]'
|
|
|
|
# Logger function.
|
|
#
|
|
function Write-Log {
|
|
$date = Get-Date -Format "yyyy-MM-dd HH:mm:ss.fff"
|
|
if ( -not($NoLog)) { Add-Content -Path $LogFilePath -Value "$date $args" }
|
|
if ($Info) { Write-Host $args }
|
|
}
|
|
|
|
# Checks the -Remote parameter.
|
|
#
|
|
# It is mandatory.
|
|
if ( -not("$Remote")) { Write-Host $Message['Usage']; exit 1 }
|
|
# The closing / is necessary.
|
|
$Remote = $Remote.TrimEnd('/') + '/'
|
|
# Must be well-formed and http(s).
|
|
if ( -not([uri]::IsWellFormedUriString("$Remote", 'Absolute')) -or -not(([uri] "$Remote").Scheme -in 'http', 'https')) {
|
|
Write-Host $Message['Bad URI']; exit 1 }
|
|
# Must be IPv4 or resolvable.
|
|
if ( -not(([uri]"$Remote").Host -match "^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$" -and [bool](([uri]"$Remote").Host -as [ipaddress]))) {
|
|
# It is resolvable?
|
|
try { Resolve-DnsName -Name ([uri]"$Remote").Host -ErrorAction Stop | Out-Null }
|
|
catch { Write-Host $Message['Bad DNS']; exit 1 }
|
|
}
|
|
#
|
|
# We've a somewhat checked remote.
|
|
|
|
# Checks the -Local parameter.
|
|
#
|
|
# Must be an existing, writable folder.
|
|
if ( -not("$Local")) { $Local = $PSScriptRoot }
|
|
if ( -not(Test-Path -LiteralPath "$Local" -pathType Container)) { Write-Host $Message['Bad folder']; exit 1 }
|
|
# Can we write into?
|
|
try {
|
|
$testfile = $Local + '\' + [guid]::NewGuid() + '.tmp'
|
|
[io.file]::OpenWrite("$testfile").close()
|
|
Remove-Item -ErrorAction SilentlyContinue "$testfile" }
|
|
catch { Write-Host $Message['Bad folder']; exit 1 }
|
|
#
|
|
# We've a somewhat checked local folder.
|
|
|
|
# Decodes the provided -Pass (if any).
|
|
#
|
|
if ("$Pass") {
|
|
try { $Pass = ([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($Pass))) }
|
|
catch { Write-Host $Message['Unable to decode']; exit 1 }
|
|
}
|
|
#
|
|
# We've a decoded (or empty) password.
|
|
|
|
# Initializes the log file.
|
|
#
|
|
$LogFilePath = $Local + '\' + (Get-Item $PSCommandPath ).Basename + (Get-Date -Format "-yyyyMMdd-HHmmss") +'.log'
|
|
Write-Log $Message['Started']
|
|
#
|
|
# We've the log file ready to use.
|
|
|
|
# Prepares the Authorization header from provided credentials (if any).
|
|
#
|
|
$Headers = ''
|
|
if ("$User" ) {
|
|
$encoded = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes("$($User):$($Pass)"))
|
|
$Headers = @{ Authorization = "Basic $encoded" }
|
|
}
|
|
# We've an Authorization header ready to use for Webrequests.
|
|
|
|
# Let's get the directory index from the remote source.
|
|
#
|
|
$response = ''
|
|
try {
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
if ("$Headers") {$response = (Invoke-WebRequest -Uri "$Remote" -Headers $Headers -UseBasicParsing) }
|
|
else {$response = (Invoke-WebRequest -Uri "$Remote" -UseBasicParsing ) }
|
|
}
|
|
catch { Write-Log $Message['Unable fetchdir'] "$Remote" $_.Exception.Response.StatusCode.Value__ $_.Exception.Response.StatusDescription; exit 1 }
|
|
$files = @($response.Links.HREF | select -skip 1)
|
|
#
|
|
# We send a warning if it is empty.
|
|
#
|
|
if ($files.Count -eq 0) { Write-Log $Message['Empty filelist'] "$Remote" }
|
|
#
|
|
# We've the list of remote files.
|
|
|
|
# Processes the remote files in a row, one after the other.
|
|
#
|
|
foreach ($file in $files) {
|
|
#
|
|
# Let's get the parameters of the remote object. On error we send a warning and move on.
|
|
#
|
|
$remoteHeaders = ''
|
|
try {
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
if ("$Headers") { $remoteHeaders = (Invoke-WebRequest -Uri ("$Remote" + "$file") -Headers $Headers -Method Head -UseBasicParsing ).Headers }
|
|
else { $remoteHeaders = (Invoke-WebRequest -Uri ("$Remote" + "$file") -Method Head -UseBasicParsing).Headers }
|
|
}
|
|
catch { Write-Log $Message['Unable to stat remote'] ("$Remote" + "$file") $_.Exception.Message; continue }
|
|
$remoteDate = $remoteHeaders['Last-Modified']
|
|
$remoteSize = $remoteHeaders['Content-Length']
|
|
$remoteType = $remoteHeaders['Content-Type']
|
|
#
|
|
# If the remote object is a folder we send a warning and move on.
|
|
#
|
|
if ("$remoteType" -eq 'text/directory') { Write-Log $Message['Is a folder'] ("$Remote" + "$file"); continue }
|
|
#
|
|
# If we've a local object and it is a folder we send a warning and move on.
|
|
#
|
|
if (Test-Path -LiteralPath "$Local\$file" -PathType Container) { Write-Log $Message['Collision array'] "$Local\$file"; continue }
|
|
#
|
|
# We've an existing local file?
|
|
#
|
|
if (Test-Path -LiteralPath "$Local\$file" -PathType Leaf) {
|
|
$localDate = (Get-Item -LiteralPath ("$Local" + '\' + "$file")).LastWriteTime.DateTime
|
|
$localSize = (Get-Item -LiteralPath ("$Local" + '\' + "$file")).Length
|
|
#
|
|
# If the local file is newer than remote we don't replace it, but we send a warning if the sizes are different.
|
|
#
|
|
if ((Get-Date $localDate) -gt (Get-Date $remoteDate)) {
|
|
if ( $localSize -ne $remoteSize ) { Write-Log $Message['Local newer'] $file }
|
|
continue
|
|
}
|
|
}
|
|
#
|
|
# OK, we decided to download the remote file.
|
|
# On failure, we'll try again a few times.
|
|
#
|
|
for ($i = 1; $i -le $MaxTries; $i++) {
|
|
try {
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
if ("$Headers") { Invoke-WebRequest -Uri ("$Remote" + "$file") -Headers $Headers -OutFile ($Local + '\' + $file) }
|
|
else { Invoke-WebRequest -Uri ("$Remote" + "$file") -OutFile ($Local + '\' + $file) }
|
|
#
|
|
Write-Log $Message['Downloaded'] ("$Remote" + "$file")
|
|
#
|
|
# Checks the size of the downloaded file, stops trying if it is OK.
|
|
#
|
|
$localSize = (Get-Item -LiteralPath ("$Local" + '\' + "$file")).Length
|
|
if ( $localSize -eq $remoteSize ) {
|
|
#
|
|
# We send a warning on small files (except the logs).
|
|
#
|
|
if ($localSize -lt $SmallSize -and (Get-Item ("$Local" + "\" + "$file")).Extension -notin ('.log')) {
|
|
Write-Log $Message['Small file'] ("$Local" + "\" + "$file") }
|
|
break
|
|
}
|
|
#
|
|
Write-Log $Message['Size mismatch'] $Local\$file $localSize $remoteSize
|
|
}
|
|
catch { Write-Log $Message['Unable to download'] ("$Remote" + "$file") $_.Exception.Message }
|
|
#
|
|
# Waits before retrying.
|
|
#
|
|
Start-Sleep -Seconds $WaitRetry
|
|
}
|
|
}
|
|
#
|
|
# That's all.
|
|
#
|
|
Write-Log $Message['Finished']
|