Fixed rclone

This commit is contained in:
Brian Zalewski 2023-07-16 10:10:04 +00:00
parent f378cf58e7
commit 9ee1459669
16 changed files with 184 additions and 52 deletions

View file

@ -19,10 +19,10 @@
#
# | Mount Location | Description |
# |-----------------------|-----------------------------------------------------------------------------------------------------------------------|
# | `/mnt/s3-private` | Private system-wide bucket used for any private files that should not be able to be accessed publicly over HTTPS |
# | `/mnt/s3-public` | Public system-wide bucket that can be accessed by anyone over HTTPS with the bucket's URL (provided by CloudFlare R2) |
# | `/mnt/s3-docker` | Private system-wide bucket used for storing Docker-related backups / files |
# | `/mnt/s3-system` | Private system-wide bucket similar to `/mnt/s3-private` but intended for system file backups |
# | `/mnt/Private` | Private system-wide bucket used for any private files that should not be able to be accessed publicly over HTTPS |
# | `/mnt/Public` | Public system-wide bucket that can be accessed by anyone over HTTPS with the bucket's URL (provided by CloudFlare R2) |
# | N/A | Private system-wide bucket used for storing Docker-related backups / files |
# | N/A | Private system-wide bucket similar to `/mnt/Private` but intended for system file backups |
# | `$HOME/.local/mnt/s3` | Private user-specific bucket (used for backing up application settings) |
#
# ## Permissions
@ -52,7 +52,7 @@
if command -v rclone > /dev/null; then
logg info 'Ensuring /var/cache/rclone exists'
sudo mkdir -p /var/cache/rclone
sudo chmod 700 /var/cache/rclone
sudo chmod 770 /var/cache/rclone
# Add user / group with script in ~/.local/bin/add-user, if it is available
if command -v add-user > /dev/null; then
sudo add-user rclone
@ -61,7 +61,7 @@ if command -v rclone > /dev/null; then
logg info 'Ensuring /var/log/rclone exists'
sudo mkdir -p /var/log/rclone
sudo chmod 700 /var/log/rclone
sudo chmod 770 /var/log/rclone
sudo chown -Rf rclone:rclone /var/log/rclone
logg info 'Adding ~/.local/bin/rclone-mount to /usr/local/bin'
@ -70,10 +70,11 @@ if command -v rclone > /dev/null; then
logg info 'Adding ~/.config/rclone/rcloneignore to /etc/rcloneignore'
sudo cp -f "${XDG_CONFIG_HOME:-$HOME/.config}/rclone/rcloneignore" /etc/rcloneignore
sudo chmod 644 /etc/rcloneignore
sudo chmod 640 /etc/rcloneignore
logg info 'Adding ~/.config/rclone/system-rclone.conf to /etc/rclone.conf'
sudo cp -f "${XDG_CONFIG_HOME:-$HOME/.config}/rclone/system-rclone.conf" /etc/rclone.conf
sudo chmod 640 /etc/rclone.conf
### Add / configure service files
if [ -d /etc/systemd/system ]; then

View file

@ -18,7 +18,7 @@
#
# If CloudFlare R2 credentials are provided, Samba is configured to store its shared files in the Rclone mounts so that your
# Samba shares are synchronized to the S3 buckets. If not, new folders are created. Either way, the folder / symlink that the
# shares host data from are stored at `/mnt/share-private` and `/mnt/share-public` (*Note: Different paths are used on macOS*).
# shares host data from are stored at `/mnt/Private` and `/mnt/Public` (*Note: Different paths are used on macOS*).
#
# 1. The **public** share (named "Public") can be accessed by anyone (including write permissions with the default settings)
# 2. The **private** share (named "Private") can be accessed by specifying the PAM credentials of anyone who has an account that is included in the `sambausers` group
@ -54,6 +54,11 @@
### Configure Samba server
if command -v smbd > /dev/null; then
# Add user / group with script in ~/.local/bin/add-user, if it is available
if command -v add-user > /dev/null; then
sudo add-user rclone
fi
### Define share locations
if [ -d /Applications ] && [ -d /System ]; then
### macOS does not have `/mnt` folder so use `/Volumes` location
@ -61,29 +66,49 @@ if command -v smbd > /dev/null; then
else
MNT_FOLDER='mnt'
fi
PRIVATE_CLOUD="/$MNT_FOLDER/Cloud (Private)"
PUBLIC_CLOUD="/$MNT_FOLDER/Cloud (Public)"
PRIVATE_SHARE="/$MNT_FOLDER/Network Share (Private)"
PUBLIC_SHARE="/$MNT_FOLDER/Network Share (Public)"
### Ensure private Samba directory / symlink exists
if [ -d "$PRIVATE_CLOUD" ] && [ ! -d "$PRIVATE_SHARE" ]; then
sudo ln -s "$PRIVATE_CLOUD" "$PRIVATE_SHARE"
else
sudo mkdir -p "$PRIVATE_SHARE"
fi
### Ensure public Samba directory / symlink exists
if [ -d "$PUBLIC_CLOUD" ] && [ ! -d "$PUBLIC_SHARE" ]; then
sudo ln -s "$PUBLIC_CLOUD" "$PUBLIC_SHARE"
else
sudo mkdir -p "$PUBLIC_SHARE"
fi
PRIVATE_SHARE="/$MNT_FOLDER/Private"
PUBLIC_SHARE="/$MNT_FOLDER/Public"
### Copy the Samba server configuration file
if [ -d /Applications ] && [ -d /System ]; then
logg warn 'TODO Add logic that applies the Samba configuration for macOS'
logg info 'Ensuring /Volumes for S3 buckets and Samba shares are configured'
logg info "Ensuring $PRIVATE_SHARE is created"
sudo mkdir -p "$PRIVATE_SHARE"
sudo chmod 770 "$PRIVATE_SHARE"
sudo chown -Rf rclone:rclone "$PRIVATE_SHARE"
logg info "Ensuring $PUBLIC_SHARE is created"
sudo mkdir -p "$PUBLIC_SHARE"
sudo chmod 775 "$PUBLIC_SHARE"
sudo chown -Rf rclone:rclone "$PUBLIC_SHARE"
logg info "Ensuring $HOME/Public is created"
mkdir -p "$HOME/Public"
chmod 775 "$HOME/Public"
chown -Rf {{ .user.username }}:rclone "$HOME/Public"
sudo sharing -a "$PRIVATE_SHARE" -S "Private (System)" -n "Private (System)" -g 000 -s 001 -E 1 -R 1 && logg success "Configured $PRIVATE_SHARE as a private Samba share"
sudo sharing -a "$PUBLIC_SHARE" -S "Public (System)" -n "Public (System)" -g 001 -s 001 -E 1 -R 0 && logg success "Configured $PUBLIC_SHARE as a public Samba share"
sudo sharing -a "$HOME/Public" -S "Public (User)" -n "Public (User)" -g 001 -s 001 -E 1 -R 0 && logg success "Configured $HOME/Public as a public Samba share"
else
logg info 'Ensuring /mnt for S3 buckets and Samba shares are configured'
logg info "Ensuring $PRIVATE_SHARE is created"
sudo mkdir -p "$PRIVATE_SHARE"
sudo chmod 770 "$PRIVATE_SHARE"
sudo chown -Rf rclone:rclone "$PRIVATE_SHARE"
logg info "Ensuring $PUBLIC_SHARE is created"
sudo mkdir -p "$PUBLIC_SHARE"
sudo chmod 775 "$PUBLIC_SHARE"
sudo chown -Rf rclone:rclone "$PUBLIC_SHARE"
logg info "Ensuring $HOME/Public is created"
mkdir -p "$HOME/Public"
chmod 775 "$HOME/Public"
chown -Rf {{ .user.username }}:rclone "$HOME/Public"
logg info "Copying Samba server configuration to /etc/samba/smb.conf"
sudo cp -f "${XDG_CONFIG_HOME:-$HOME/.config}/samba/config" "/etc/samba/smb.conf"

View file

@ -1,3 +1,3 @@
{{- if and (or (and (stat (joinPath .host.home ".config" "age" "chezmoi.txt")) (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_ID"))) (env "CLOUDFLARE_R2_ID")) (or (and (stat (joinPath .host.home ".config" "age" "chezmoi.txt")) (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_SECRET"))) (env "CLOUDFLARE_R2_SECRET")) (ne .user.cloudflare.r2 "") -}}
/mnt/s3-private
/{{- if eq .host.distro.id "darwin" -}}Volumes{{- else -}}mnt{{- end -}}/Private
{{- end -}}

View file

@ -1,3 +1,3 @@
{{- if and (or (and (stat (joinPath .host.home ".config" "age" "chezmoi.txt")) (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_ID"))) (env "CLOUDFLARE_R2_ID")) (or (and (stat (joinPath .host.home ".config" "age" "chezmoi.txt")) (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_SECRET"))) (env "CLOUDFLARE_R2_SECRET")) (ne .user.cloudflare.r2 "") -}}
/mnt/s3-public
/{{- if eq .host.distro.id "darwin" -}}Volumes{{- else -}}mnt{{- end -}}/Public
{{- end -}}

View file

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>KeepAlive</key>
<true/>
<key>Label</key>
<string>rclone.private</string>
<key>ProgramArguments</key>
<array>
<string>/usr/local/bin/rclone-mount</string>
<string>rclone</string>
<string>rclone</string>
<string>Private</string>
</array>
<key>RunAtLoad</key>
<true/>
<key>StandardErrorPath</key>
<string>/var/log/rclone/private-daemon.error.log</string>
<key>StandardOutPath</key>
<string>/var/log/rclone/private-daemon.debug.log</string>
</dict>
</plist>

View file

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>KeepAlive</key>
<true/>
<key>Label</key>
<string>rclone.public</string>
<key>ProgramArguments</key>
<array>
<string>/usr/local/bin/rclone-mount</string>
<string>rclone</string>
<string>rclone</string>
<string>Public</string>
</array>
<key>RunAtLoad</key>
<true/>
<key>StandardErrorPath</key>
<string>/var/log/rclone/public-daemon.error.log</string>
<key>StandardOutPath</key>
<string>/var/log/rclone/public-daemon.debug.log</string>
</dict>
</plist>

View file

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>KeepAlive</key>
<true/>
<key>Label</key>
<string>rclone.public</string>
<key>ProgramArguments</key>
<array>
<string>/usr/local/bin/rclone-mount</string>
<string>user</string>
<string>{{ .user.username }}</string>
<string>s3</string>
</array>
<key>RunAtLoad</key>
<true/>
<key>StandardErrorPath</key>
<string>/Users/{{ .user.username }}/.local/share/rclone/error.log</string>
<key>StandardOutPath</key>
<string>/Users/{{ .user.username }}/.local/share/rclone/debug.log</string>
</dict>
</plist>

View file

@ -14,4 +14,19 @@ if [ -d /Applications ] && [ -d /System ]; then
sudo cp -f "$HOME/Library/LaunchDaemons/com.cloudflare.warp.plist" '/Library/LaunchDaemons/com.cloudflare.warp.plist'
sudo launchctl load "/Library/LaunchDaemons/com.cloudflare.warp.plist"
fi
### Enable Rclone mounts
if [ -f "$HOME/Library/LaunchDaemons/rclone.private.plist" ] && [ ! -f "/Library/LaunchDaemons/rclone.private.plist" ]; then
sudo mkdir -p /Library/LaunchDaemons
sudo cp -f "$HOME/Library/LaunchDaemons/rclone.private.plist" '/Library/LaunchDaemons/rclone.private.plist'
sudo launchctl load '/Library/LaunchDaemons/rclone.private.plist'
fi
if [ -f "$HOME/Library/LaunchDaemons/rclone.public.plist" ] && [ ! -f "/Library/LaunchDaemons/rclone.public.plist" ]; then
sudo mkdir -p /Library/LaunchDaemons
sudo cp -f "$HOME/Library/LaunchDaemons/rclone.public.plist" '/Library/LaunchDaemons/rclone.public.plist'
sudo launchctl load '/Library/LaunchDaemons/rclone.public.plist'
fi
if [ -f "$HOME/Library/LaunchDaemons/rclone.user.plist" ]; then
launchctl load "$HOME/Library/LaunchDaemons/rclone.user.plist"
fi
fi

View file

@ -18,7 +18,7 @@ tee -a "$CONFIG_FILE" > /dev/null <<EOT
[{{ .user.username}}-s3]
access_key_id = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_ID")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_ID" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_ID" -}}{{ end }}
acl = private
endpoint = {{ .user.cloudflare.r2 }}.r2.cloudflarestorage.com/user
endpoint = {{ .user.cloudflare.r2 }}.r2.cloudflarestorage.com
provider = Cloudflare
region = auto
secret_access_key = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_SECRET")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_SECRET" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_SECRET" -}}{{ end }}

View file

@ -2,25 +2,25 @@
[docker]
access_key_id = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_ID")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_ID" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_ID" -}}{{ end }}
acl = private
endpoint = {{ .user.cloudflare.r2 }}.r2.cloudflarestorage.com/docker
endpoint = {{ .user.cloudflare.r2 }}.r2.cloudflarestorage.com
env_auth = false
provider = Cloudflare
secret_access_key = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_SECRET")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_SECRET" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_SECRET" -}}{{ end }}
type = s3
[private]
[Private]
access_key_id = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_ID")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_ID" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_ID" -}}{{ end }}
acl = private
endpoint = {{ .user.cloudflare.r2 }}.r2.cloudflarestorage.com/private
endpoint = {{ .user.cloudflare.r2 }}.r2.cloudflarestorage.com
env_auth = false
provider = Cloudflare
secret_access_key = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_SECRET")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_SECRET" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_SECRET" -}}{{ end }}
type = s3
[public]
[Public]
access_key_id = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_ID")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_ID" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_ID" -}}{{ end }}
acl = public-read
endpoint = {{ .user.cloudflare.r2 }}.r2.cloudflarestorage.com/public
endpoint = {{ .user.cloudflare.r2 }}.r2.cloudflarestorage.com
env_auth = false
provider = Cloudflare
secret_access_key = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_SECRET")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_SECRET" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_SECRET" -}}{{ end }}
@ -29,7 +29,7 @@ type = s3
[system]
access_key_id = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_ID")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_ID" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_ID" -}}{{ end }}
acl = private
endpoint = {{ .user.cloudflare.r2 }}.r2.cloudflarestorage.com/system
endpoint = {{ .user.cloudflare.r2 }}.r2.cloudflarestorage.com
env_auth = false
provider = Cloudflare
secret_access_key = {{ if (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "CLOUDFLARE_R2_SECRET")) }}{{- includeTemplate "secrets/CLOUDFLARE_R2_SECRET" | decrypt | trim -}}{{ else }}{{- env "CLOUDFLARE_R2_SECRET" -}}{{ end }}

View file

@ -1,6 +1,6 @@
**/.**
**/_**
**/*.db
**/*.box
**/Network Trash Folder**
**/node_modules**
.*
_*
*.box
*.db
node_modules/**
Network Trash Folder/**

View file

@ -5,8 +5,8 @@ After=network-online.target
[Service]
Type=simple
User=rclone
ExecStart=/usr/local/bin/rclone-mount "rclone" "rclone" "Cloud (Private)"
ExecStop=/bin/fusermount -u "/mnt/Cloud (Private)"
ExecStart=/usr/local/bin/rclone-mount "rclone" "rclone" "Private"
ExecStop=/bin/fusermount -u "/mnt/Private"
Restart=always
RestartSec=10

View file

@ -5,8 +5,8 @@ After=network-online.target
[Service]
Type=simple
User=rclone
ExecStart=/usr/local/bin/rclone-mount "rclone" "rclone" "Cloud (Public)"
ExecStop=/bin/fusermount -u "/mnt/Cloud (Public)"
ExecStart=/usr/local/bin/rclone-mount "rclone" "rclone" "Public"
ExecStop=/bin/fusermount -u "/mnt/Public"
Restart=always
RestartSec=10

View file

@ -28,7 +28,7 @@
comment = Authenticated share with read/write capabilities (backed up to S3)
follow symlinks = yes
guest only = no
path = /mnt/Network Share (Private)
path = /mnt/Private
public = no
unix extension = yes
valid users = @sambausers
@ -40,7 +40,16 @@
comment = Public folder provided for file sharing on the LAN (backed up to S3)
follow symlinks = no
guest only = yes
path = /mnt/Network Share (Public)
path = /mnt/Public
public = yes
writable = yes
[User]
browsable = yes
comment = Public user home directory folder for file sharing on the LAN
follow symlinks = no
guest only = yes
path = /home/{{ .user.name }}/Public
public = yes
writable = yes

View file

@ -33,7 +33,9 @@ fi
### Ensure folders exist
for FOLDER in "$CACHE_FOLDER" "$CONFIG_FOLDER" "$LOG_FOLDER" "$MOUNT_PATH"; do
if [ ! -d "$FOLDER" ]; then
mkdir -p "$FOLDER" || echo "ERROR: Need permissions for $FOLDER"
sudo mkdir -p "$FOLDER" || echo "ERROR: Need permissions for $FOLDER"
sudo chmod 755 "$FOLDER"
sudo chown -Rf rclone:rclone "$FOLDER"
fi
done
@ -44,7 +46,10 @@ if [ ! -f "$RCLONE_IGNORE" ] && [ -f "/etc/rcloneignore" ]; then
fi
### Mount
/usr/bin/rclone --config="$CONFIG_FOLDER/rclone.conf" \
MOUNT_LOWERCASE="$(echo "$MOUNT" | tr "[:upper:]" "[:lower:]")"
unset AWS_CA_BUNDLE
export PATH="$PATH:/usr/local/bin:/usr/bin"
sudo rclone --config="$CONFIG_FOLDER/rclone.conf" \
mount \
--cache-tmp-upload-path="$CACHE_FOLDER/$MOUNT-upload" \
--cache-chunk-path="$CACHE_FOLDER/$MOUNT-chunks" \
@ -52,6 +57,8 @@ fi
--cache-writes \
--cache-dir="$CACHE_FOLDER/$MOUNT-vfs" \
--cache-db-path="$CACHE_FOLDER/$MOUNT-db" \
--daemon \
--noapplexattr \
--log-file="$LOG_FILE" \
--no-modtime \
--drive-use-trash \
@ -60,6 +67,7 @@ fi
--bwlimit=40M \
--dir-cache-time=60m \
--vfs-cache-mode full \
--volname "$MOUNT" \
--cache-info-age=60m \
--filter-from="$RCLONE_IGNORE"
"$MOUNT":/ "$MOUNT_PATH"
--filter-from="$RCLONE_IGNORE" \
"$MOUNT_LOWERCASE":"$MOUNT_LOWERCASE" "$MOUNT_PATH"

View file

@ -2154,6 +2154,7 @@ softwarePackages:
_github: https://github.com/imp/dnsmasq
_home: null
_name: dnsmasq
_service: dnsmasq
ansible: professormanhattan.dnsmasq
apt: dnsmasq
brew: dnsmasq
@ -6101,6 +6102,7 @@ softwarePackages:
_github: null
_home: https://www.nomadproject.io/
_name: HashiCorp Nomad
_service: nomad
ansible: professormanhattan.nomad
brew: nomad
choco: nomad
@ -8238,6 +8240,7 @@ softwarePackages:
_github: null
_home: null
_name: Stubby
_service: stubby
_post:brew:darwin: bash "$(brew --prefix stubby)/sbin/stubby-setdns-macos.sh" && brew services start stubby
brew:darwin: stubby
surge:
@ -8861,6 +8864,7 @@ softwarePackages:
- privoxy
_home: https://www.privoxy.org/
_name: Privoxy
_service: privoxy
apt: privoxy
brew: privoxy
dnf: privoxy
@ -8873,6 +8877,7 @@ softwarePackages:
_github: https://github.com/TheTorProject
_home: https://www.torproject.org/
_name: TOR
_service: tor
ansible:linux: professormanhattan.tor
apt: tor
brew: tor