seedbox_sync.py 2.5 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758
  1. # rsync files from a seedbox to a local machine, exactly once, over SSH.
  2. #
  3. # Why?
  4. # sonarr requires that any Remote Path Mappings have a local path reflecting its contents. This can be done with NFS or SSHFS, but those are difficult to set up in containers, and get wonky when the remote server reboots.
  5. # rsync over SSH + cron doesn't care if the remote machine reboots, + easily runs in a container.
  6. # How?
  7. # Usage: sonarr_sync.py my-seedbox /seedbox/path/to/data /local/working /local/metadata /local/data
  8. # - Get all file names in my-seedbox:/seedbox/path/to/data
  9. # - Get all previously processed file names in /local/metadata
  10. # - Diff the above to get newly added files
  11. # - For each new file:
  12. # - Copy file from my-seedbox to /local/working (used in case of transfer failure)
  13. # - Add file name to /local/metadata
  14. # - Move file to /local/data
  15. # */1 * * * * /usr/bin/run-one /usr/bin/python3 /path/to/seedbox_sync.py <seedbox host> /seedbox/path/to/completed/ /local/path/to/downloading /local/path/to/processed /local/path/to/ready 2>&1 | /usr/bin/logger -t seedbox
  16. # Or run it in a k8s cronjob.
  17. import subprocess
  18. import sys
  19. if len(sys.argv) != 6:
  20. print("One or more args are undefined")
  21. sys.exit(1)
  22. host, host_data_path, local_working_path, local_metadata_path, local_data_path = sys.argv[1:6]
  23. r = subprocess.run(["ssh", host, "bash", "-c", f"IFS=$'\n'; ls {host_data_path}"], stdout=subprocess.PIPE, check=True)
  24. available = {f for f in r.stdout.decode().split('\n') if f}
  25. # There's better ways to list a dir locally, but using bash +ls again avoids any possible formatting discrepencies.
  26. r = subprocess.run(["bash", "-c", f"IFS=$'\n'; ls {local_metadata_path}"], stdout=subprocess.PIPE, check=True)
  27. processed = {f for f in r.stdout.decode().split('\n') if f}
  28. new = available - processed
  29. for new_file in new:
  30. # Be super cautious about empty file names, wouldn't want to `rm -rf` a folder by accident
  31. if not new_file:
  32. continue
  33. print(f"Processing: {new_file}")
  34. subprocess.run(["rsync", "-rsvv", f'{host}:{host_data_path}/{new_file}', f'{local_working_path}'], check=True)
  35. subprocess.run(["touch", f'{local_metadata_path}/{new_file}'], check=True)
  36. print(f"Moving to ready: {new_file}")
  37. try:
  38. # rsync here is probably overkill
  39. subprocess.run(["rsync", "-r", f'{local_working_path}/{new_file}', f'{local_data_path}'], check=True)
  40. except:
  41. subprocess.run(["rm", f'{local_metadata_path}/{new_file}'], check=True)
  42. raise
  43. subprocess.run(["rm", "-rf", f'{local_working_path}/{new_file}'], check=True)