Skip to content

Instantly share code, notes, and snippets.

@pwhe23

pwhe23/Sync.cs

Created July 9, 2014 15:27
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save pwhe23/0e7d526e824ddb986ecb to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save pwhe23/0e7d526e824ddb986ecb to your computer and use it in GitHub Desktop.
Download ZIP
Sync FtpClient
Raw
Sync.cs
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
namespace System.Net.FtpClient
{
public class Sync
{
private FtpClient _ftp;
public Sync()
{
Filter = new FileFilter();
HashFilter = new FileFilter();
}
public string Host { get; set; }
public string Username { get; set; }
public string Password { get; set; }
public bool Preview { get; set; }
public bool ShowDebug { get; set; }
public string LocalPath { get; set; }
public string RemotePath { get; set; }
public FileFilter Filter { get; set; }
public FileFilter HashFilter { get; set; }
public Action<string> Writer { get; set; }
public List<FileData> Execute()
{
if (Preview) WriteLine("PREVIEW MODE");
using (_ftp = new FtpClient())
{
_ftp.Credentials = new NetworkCredential(Username, Password);
_ftp.Host = Host;
_ftp.Connect();
var root = new DirectoryInfo(LocalPath);
Write("BuildLocalFileList: " + LocalPath + (ShowDebug ? "\r\n" : " "));
var local = BuildLocalFileList(root.FullName, "");
if (!ShowDebug) WriteLine("");
Write("BuildFtpFileList: " + RemotePath + (ShowDebug ? "\r\n" : " "));
var remote = BuildFtpFileList(RemotePath, "");
if (!ShowDebug) WriteLine("");
_ftp.SetWorkingDirectory(RemotePath);
var changes = CompareFiles(local.ToDictionary(x => x.Path.ToLower(), x => x),
remote.ToDictionary(x => x.Path.ToLower(), x => x));
foreach (var file in changes)
{
if (file.Delete)
{
WriteLine("DELETE: {0}", file.Path);
if (!Preview) _ftp.DeleteFile(file.Path);
}
else if (file.Changed)
{
WriteLine(ShowDebug ? "UPLOAD: {0} [{1},{2}] {3}" : "UPLOAD: {0}", file.Path, file.Size, file.Hash, file.Info);
if (!Preview) Upload(root.FullName + file.Path.Replace("/", @"\"), file.Path);
}
}
return changes;
}
}
private List<FileData> BuildLocalFileList(string root, string path)
{
var list = new List<FileData>();
var dir = new DirectoryInfo(Path.Combine(root, path));
//recurse child dirs
foreach (var child in dir.GetDirectories())
{
var name = Path.Combine(child.FullName.Replace(root, ""), "").Replace('\\', '/');
if (!Filter.Check(name))
continue;
list.AddRange(BuildLocalFileList(root, Path.Combine(path, child.Name)));
}
//build file list
foreach (var file in dir.GetFiles())
{
var name = file.FullName.Replace(root, "").Replace('\\', '/');
if (!Filter.Check(name))
continue;
var hash = GetMd5Hash(File.ReadAllBytes(file.FullName));
list.Add(new FileData
{
Path = name,
Size = file.Length,
Hash = hash,
});
if (ShowDebug)
DebugLine("LOCALFILE: {0} ({1},{2})", name, file.Length, hash);
else
Write(".");
}
return list;
}
private List<FileData> BuildFtpFileList(string root, string path)
{
var list = new List<FileData>();
var dirs = new List<string>();
DebugLine("FTPDIR: " + root + path);
_ftp.SetWorkingDirectory(root + path);
var files = _ftp.GetListing();
//build file list
foreach (var file in files)
{
var name = path + file.Name;
if (!Filter.Check(name))
continue;
if (file.Type == FtpFileSystemObjectType.File)
{
list.Add(new FileData
{
Path = name,
Size = file.Size,
});
if (ShowDebug)
DebugLine("FTPFILE: {0} ({1})", name, file.Size);
else
Write(".");
}
else
{
dirs.Add(name);
}
}
//recurse child dirs
foreach (var child in dirs)
{
var name = child + "/";
if (!Filter.Check(name))
continue;
list.AddRange(BuildFtpFileList(root, name));
}
return list;
}
private List<FileData> CompareFiles(Dictionary<string, FileData> source, Dictionary<string, FileData> dest)
{
var list = new List<FileData>();
//find changes
foreach (var file in source)
{
//Make sure file exists
if (!dest.ContainsKey(file.Key))
{
list.Add(new FileData
{
Path = file.Value.Path,
Size = file.Value.Size,
Hash = file.Value.Hash,
Changed = true,
Info = "NEW",
});
continue;
}
//make sure it is the same size
else if (file.Value.Size != dest[file.Key].Size)
{
list.Add(new FileData
{
Path = file.Value.Path,
Size = file.Value.Size,
Hash = file.Value.Hash,
Changed = true,
Info = "SIZE:" + dest[file.Key].Size,
});
continue;
}
//check hash size
var path = dest[file.Key].Path;
if (!HashFilter.Check(path))
continue;
Debug("FTPHASH: {0}", path);
var hash = GetMd5Hash(Download(RemotePath + path));
DebugLine(" ({0})", hash);
if (file.Value.Hash != hash)
{
list.Add(new FileData
{
Path = file.Value.Path,
Size = file.Value.Size,
Hash = file.Value.Hash,
Changed = true,
Info = "HASH:" + hash,
});
}
}
//find deletes
foreach (var file in dest)
{
//make sure it exists in source
if (!source.ContainsKey(file.Key))
{
list.Add(new FileData
{
Path = file.Value.Path,
Size = file.Value.Size,
Delete = true,
});
}
}
return list;
}
//REF: http://stackoverflow.com/a/13806183/366559
private static string GetMd5Hash(byte[] input)
{
var md5Hash = MD5.Create();
// Convert the input string to a byte array and compute the hash.
var data = md5Hash.ComputeHash(input);
// Create a new Stringbuilder to collect the bytes
// and create a string.
var sBuilder = new StringBuilder();
// Loop through each byte of the hashed data
// and format each one as a hexadecimal string.
for (var i = 0; i < data.Length; i++)
{
sBuilder.Append(data[i].ToString("x2"));
}
// Return the hexadecimal string.
return sBuilder.ToString();
}
private void Write(object msg, params object[] args)
{
var txt = (msg == null ? ""
: args.Length == 0 ? msg.ToString()
: String.Format(msg.ToString(), args));
if (Writer != null)
Writer(txt);
}
private void WriteLine(object msg, params object[] args)
{
Write(msg.Or() + Environment.NewLine, args);
}
private void Debug(object msg, params object[] args)
{
if (!ShowDebug)
return;
Write(msg, args);
}
private void DebugLine(object msg, params object[] args)
{
if (!ShowDebug)
return;
WriteLine(msg, args);
}
private void Upload(string source, string dest)
{
var path = Path.GetDirectoryName(dest).Replace(RemotePath, "");
if (!_ftp.DirectoryExists(path))
_ftp.CreateDirectory(path, true);
using (
Stream istream = new FileStream(source, FileMode.Open, FileAccess.Read),
ostream = _ftp.OpenWrite(dest.Replace(RemotePath, ""), FtpDataType.Binary))
{
var buf = new byte[8192];
int read;
try
{
while ((read = istream.Read(buf, 0, buf.Length)) > 0)
{
ostream.Write(buf, 0, read);
}
}
finally
{
ostream.Close();
istream.Close();
}
}
}
private byte[] Download(string file)
{
using (Stream reader = _ftp.OpenRead(file), writer = new MemoryStream())
{
var buf = new byte[8192];
try
{
int read;
while ((read = reader.Read(buf, 0, buf.Length)) > 0)
{
writer.Write(buf, 0, read);
}
return writer.ReadAllBytes();
}
finally
{
reader.Close();
writer.Close();
}
}
}
};
public class FileData
{
public string Path { get; set; }
public long Size { get; set; }
public string Hash { get; set; }
public bool Changed { get; set; }
public bool Delete { get; set; }
public string Info { get; set; }
};
public class FileFilter
{
private RegexOptions _options;
private enum MatchType
{
Include,
Exclude,
}
private class MatchItem
{
public MatchType Type;
public string Pattern;
}
private List<MatchItem> _patterns = new List<MatchItem>();
public FileFilter()
{
_options = RegexOptions.IgnoreCase;
}
public void Include(params string[] patterns)
{
_patterns.AddRange(patterns.Select(x => new MatchItem { Type = MatchType.Include, Pattern = x }));
}
public void Exclude(params string[] patterns)
{
_patterns.AddRange(patterns.Select(x => new MatchItem { Type = MatchType.Exclude, Pattern = x }));
}
public bool Check(string path)
{
var shouldInclude = false;
foreach (var item in _patterns)
{
if (item.Type == MatchType.Include && Regex.IsMatch(path, "^" + item.Pattern + "$", _options))
{
shouldInclude = true;
}
else if (item.Type == MatchType.Exclude && Regex.IsMatch(path, "^" + item.Pattern + "$", _options))
{
shouldInclude = false;
}
}
return shouldInclude;
}
};
internal static class Ext
{
public static bool HasValue(this object obj)
{
if (obj == null) return false;
if (obj is String) return HasValue(obj as String);
return true;
}
public static bool HasValue(this DateTime date)
{
return date != DateTime.MinValue;
}
public static bool HasValue(this DateTime? date)
{
return date.HasValue && date != DateTime.MinValue;
}
public static bool HasValue(this String txt)
{
return !String.IsNullOrWhiteSpace(txt);
}
public static bool In(this String txt, params string[] vals)
{
if (txt == null || vals.Length == 0) return false;
return vals.Any(txt.Is);
}
public static bool Is(this String txt, object val)
{
if (txt == null && val == null) return true;
if (txt == null || val == null) return false;
return txt.Equals(val.ToString(), StringComparison.InvariantCultureIgnoreCase);
}
public static bool Is(this Object one, Object two)
{
return Is(one == null ? String.Empty : one.ToString(), two == null ? String.Empty : two.ToString());
}
public static string Or(this Object obj, params object[] vals)
{
if (obj != null && obj.ToString().HasValue()) return obj.ToString();
foreach (var v in vals)
{
if (v != null && v.ToString().HasValue()) return v.ToString();
}
return "";
}
//REF: http://www.yoda.arachsys.com/csharp/readbinary.html
public static byte[] ReadAllBytes(this Stream stream)
{
if (stream is MemoryStream) return (stream as MemoryStream).ToArray();
var buffer = new byte[32768];
using (var ms = new MemoryStream())
{
while (true)
{
int read = stream.Read(buffer, 0, buffer.Length);
if (read <= 0) return ms.ToArray();
ms.Write(buffer, 0, read);
}
}
}
};
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment