Posts Tagged With ‘Folder Synchronization&8217


Synchronize two Folders and its subfolder contents

The below class synchronizes target folder to ‘shadow’ source folder (Including sub-folders).
It makes updates in target folder based on only delta changes in source folder structure i.e. updates/create/deletes only files which are touched in source folder. Hence preserving target folder/files’ timestamps which are not being changed in the source.

	
import java.io.File;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.List;

public class FolderSync {

	private String srcFolder = null;	
	private String targetFolder = null;
	private String timeInterval = null;

	private List srcFoldersList= new ArrayList(); // Stores source folder structure
	private List targetFoldersList=new ArrayList();  // Stores target folder structure

	/**	 
	 * @param srcFolder the Source Folder full path
	 * @param targetFolder the Target Folder full path
	 * @param timeInterval the Time interval between synchronization
	 * 
	 * */
	
	public FolderSync(String srcFolder, String targetFolder, String timeInterval ) {
		this.srcFolder = srcFolder;
		this.targetFolder = targetFolder;
		this.timeInterval = timeInterval;
	}



	public FolderSync() {
		srcFolder = null;
		targetFolder = null;
		timeInterval = null;		
	}



	/**	
	 * It returns resultant String list after performing a logical minus operation on two String list	
	 *   
	 * @return String An array of String Containing difference
	 * @param a - the String array from which we need to compare
	 * @param b - the String array to which we need to compare
	 */

	//for instance args[0] contains Source Folder, args[1] contains Target Folder (to be synchronized)
	public ArrayList findMinus(String[] a, String[] b) {

		Arrays.sort(a);
		Arrays.sort(b);

		ArrayList result = new ArrayList();

		for (int i = 0; i < a.length; i++) {
			boolean notFound = true;
			for (int j = 0; j < b.length; j++) {
				if ((a[i].compareTo(b[j])) == 0) {
					notFound = false;
					break;
				}
			}

			if (notFound) {
				result.add(a[i]);
			}

		}

		return result;

	}

	/**

	 * It copies files/folder from source folder to target folder, the files/folder which doesn't exists in target folder
	 * @param str_src_list the source String Array
	 * @param str_target_list the target String Array
	 * @param local_src_folder the local source folder String, 
	 * @param local_target_folder the local target folder  String,
	   @param isFolder boolean true for folder
	 */

	public void startCopySrc(String[] str_src_list, String[] str_target_list,
			String local_src_folder, String local_target_folder,
			boolean isFolder) {

		ArrayList result = new ArrayList();
		result.addAll(this.findMinus(str_src_list, str_target_list));

		File[] result_f = new File[result.size()];
		String[] result_str = new String[result.size()];

		for (int i = 0; i < result.size(); i++) {
			result_f[i] = new File(local_src_folder
					+ System.getProperty("file.separator") + result.get(i));
			result_str[i] = new String(result.get(i));
		}

		String copied_file_namePath;
		String copied_folder_namePath;
		File f;
		for (int i = 0; i < result.size(); i++) {
			if (!isFolder) {
				copied_file_namePath = local_target_folder
						+ System.getProperty("file.separator") + result_str[i];
				f = new File(copied_file_namePath);

				if (!f.exists()) {
					FileUtil.copyFile(result_f[i].getAbsolutePath(),
							copied_file_namePath);
				}
			} else {

				copied_folder_namePath = local_target_folder + result_str[i];
				f = new File(copied_folder_namePath);
				if (!f.exists())
					f.mkdir();// make a new folder

			}

		}
	}





	/**
	 * It deletes files/folder from target folder, the files/folder which doesn't exists in source folder
	 *
	 * 
	 * 
	 * @param strSrcList the source String Array
	 * @param strTargetList the target String Array
	 * @param localSrcFolder the local source folder String, 
	 * @param localTargetFolder the local target folder  String,
	   @param isFolder boolean true for folder
	 */

	public void startDeleteTarget(String[] strSrcList,
			String[] strTargetList, String localSrcFolder,
			String localTargetFolder, boolean isFolder){

		ArrayList result = new ArrayList();
		result.addAll(this.findMinus(strTargetList, strSrcList));

		File[] result_f = new File[result.size()];
		String[] result_str = new String[result.size()];

		for (int i = 0; i < result.size(); i++) {
			result_f[i] = new File(localSrcFolder + result.get(i));
			result_str[i] = new String(result.get(i));
		}

		String copied_file_namePath = null;
		File f = null;

		for (int i = 0; i < result.size(); i++) {
			if (!isFolder) {
				copied_file_namePath = localTargetFolder
						+ System.getProperty("file.separator") + result_str[i];
				f = new File(copied_file_namePath);
				f.delete();

			} else {

				copied_file_namePath = localTargetFolder + result_str[i];
				f = new File(copied_file_namePath);
				try {
					FileUtil.deleteDir(f);
				} catch (Exception e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				} //Delete the folder
			}

		}
	}






	/** 
	 * It compare changes in file, based on their checksum values  
	 * It requires both folder's file count and Filename is same i.e startCopySrc()  as well as startDeleteTarget()  should have already executed prior to this
	 * 
	 * @param strSrcList the source String Array
	 * @param strTargetList the target String Array
	 * @param localSrcFolder the local source folder String 
	 * @param localTargetFolder the local target folder  String
	 * 
	 * @throws Exception 
	 */
	 
	public void startSyncChecksum(String[] strSrcList, String[] strTargetList,
			String localSrcFolder, String localTargetFolder)
			throws Exception {		
		
		FileInfo src_fi=null;
		FileInfo target_fi=null;
		
		String srcFilePath=null;
		String targetFilePath=null;	
		
		java.util.Arrays.sort(strSrcList);
		java.util.Arrays.sort(strTargetList);
		
		for (int i = 0; i < strSrcList.length; i++) {
			 
			
			srcFilePath=localSrcFolder
			+ System.getProperty("file.separator") + strSrcList[i];			
			if(srcFilePath!=null)
			 src_fi = new FileInfo(srcFilePath);
			
			
			targetFilePath=localTargetFolder
			+ System.getProperty("file.separator") + strTargetList[i];
			if(targetFilePath!=null)				
			target_fi = new FileInfo(targetFilePath);

			
			if (src_fi.isFile()) {
				if (src_fi.calcChecksum() != target_fi.calcChecksum()) {
					FileUtil.copyFile(srcFilePath, targetFilePath);
				}
			}
		}

	}









	/** 	
	 * It traverses source folder and update src_folder_list member variable to reflect current source folder structure (which is expected to keep changing very frequently)
	 * @param folder
	 * 
	 * 
	 * */

	private void traverseSrcFolders(File folder) {
		if (folder.isDirectory()) {
			srcFoldersList.add(folder.toString());
			String[] children = folder.list();
			for (int i = 0; i < children.length; i++) {
				traverseSrcFolders(new File(folder, children[i]));
			}
		}
	}



	/** 	
	 * It traverses source folder and update target_folder_list member variable to reflect current target folder structure ( which is expected to keep changing very frequently)

	 * @param folder
	 * 
	 * 
	 * */
	private void traverseTargetFolders(File folder) {
		if (folder.isDirectory()) {
			targetFoldersList.add(folder.toString());
			String[] children = folder.list();
			for (int i = 0; i < children.length; i++) {
				traverseTargetFolders(new File(folder, children[i]));
			}
		}
	}





	/** 
	 * It strips off a Array String list of a preceding string.	
	 * @param tmp_folders_list the list of folders
	 * @param remove_string the string which needs to be removed
	 * @return String[] 
	 */

	private String[] setLeafFolder(List tmp_folders_list,
			String remove_string) {
		String tmp_str = null;

		for (int i = 0; i < tmp_folders_list.size(); i++) {
			tmp_str = ((tmp_folders_list.get(i).toString()
					.substring(remove_string.length())));
			tmp_folders_list.set(i, tmp_str);

		}

		String[] str_folders_list = new String[tmp_folders_list.size()];
		for (int i = 0; i < tmp_folders_list.size(); i++) {
			str_folders_list[i] = new String(tmp_folders_list.get(i));
		}

		return str_folders_list;

	}






	/** 
	 * 	
	 * It sync multilevel sub-folders just structures between source and folder	 
	 * @throws Exception
	 * 
	 * 	 * 
	 * */
	public void subFoldersSync() throws Exception {
		File s_folder = new File(this.srcFolder);
		srcFoldersList = new ArrayList();

		this.traverseSrcFolders(s_folder); // Setting src folder structure

		File t_folder = new File(this.targetFolder);
		targetFoldersList = new ArrayList();

		this.traverseTargetFolders(t_folder); // Setting target folder structure

		String src_r = this.srcFoldersList.get(0).toString();
		String target_r = this.targetFoldersList.get(0).toString();
		srcFoldersList.remove(0); // Removing root folder name from list, we don't want to sync that name to target, (e.g. source)
		targetFoldersList.remove(0); // Removing root folder name from list (e.g. target)

		String[] sf = this.setLeafFolder(srcFoldersList, src_r); // We don't want entire path name instead just leaf folder name
		String[] tf = this.setLeafFolder(targetFoldersList, target_r);

		this.startCopySrc(sf, tf, this.srcFolder, this.targetFolder, true);
		this.startDeleteTarget(sf, tf, this.srcFolder, this.targetFolder,
				true);
	}

	/**

	
	
	 * It does files/folders/subfolders synchronization 
	 * 	
	 * @throws Exception	  
	 * */

	public void startSync(){
		try {
			subFoldersSync();
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		srcFoldersList.add(""); // for the root

		//since src and target folder structure would be same after subfolder syncing, we can run loop times size of src_folders_list even on target_folders_list

		// Copy and Delete Files/Folders
		for (int i = 0; i < srcFoldersList.size(); i++) {
			String l_src = this.srcFolder + srcFoldersList.get(i);
			File s = new File(l_src);
			String[] s_list = s.list();

			String l_target = this.targetFolder + srcFoldersList.get(i);
			File t = new File(l_target);
			String[] t_list = t.list();

			this.startCopySrc(s_list, t_list, l_src, l_target, false);
			this.startDeleteTarget(s_list, t_list, l_src, l_target, false);
		}

		// Update file/folders only after source and target has identical structure 
		//TODO Make this logic independent, to give way to multi threading operation, because SyncChecksum func is a bottleneck in this application
		for (int i = 0; i < srcFoldersList.size(); i++) {
			String l_src = this.srcFolder + srcFoldersList.get(i);

			File s = new File(l_src);
			String[] s_list = s.list();

			String l_target = this.targetFolder + srcFoldersList.get(i);

			File t = new File(l_target);
			String[] t_list = t.list();

			try {
				this.startSyncChecksum(s_list, t_list, l_src, l_target);
			} catch (Exception e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}






	/**	 
	 * It runs the sync on specified interval
	
	 * */

	public void run() throws Exception {

		boolean alive = true;

		while (alive) {
			this.startSync();
			Thread.sleep(Long.parseLong(timeInterval));
			if (this.timeInterval.compareTo("0") == 0)
				alive = false; // It will run just once

		}
	}




	/** 	 
	 * Verifies and Set the argument passed	 
	 * 
	 * @param args
	 * 
	 * 
	 * */

	public FolderSync validateArguments(String args[]) {

		String tInterval = "0";

		if (args.length < 2) {
			System.err.println("Insufficient numbers of arguments");
			System.exit(0);
		}

		if (args.length > 3) {
			System.err.println("Too many arguments");
			System.exit(0);
		}

		if (args[0] != null) {
			File f = new File(args[0]);
			if (!f.exists())
				System.err.println("Source Folder doesn't exists!");
		} else {
			System.err.println("Source Folder cannot be null!");
			System.exit(0);
		}

		if (args[1] != null) {
			File f = new File(args[1]);
			if (!f.exists())
				System.err.println("Target Folder doesn't exists!");
		} else {
			System.err.println("Target Folder cannot be null!");
			System.exit(0);
		}

		if (args.length == 3) {

			try {
				Integer.parseInt(args[2]);
			} catch (NumberFormatException e) {
				System.err.println(args[2]
						+ " is a invalid time interval! -- Should be a number");
				System.exit(0);
			}

			tInterval = args[2];
		}

		FolderSync fs = new FolderSync(args[0], args[1], tInterval);

		return fs;
	}





	/**	
	 * Main function 	
	 *  
	 * Takes three arguments, two are required and third one is optional	
	 * {@code args[0]} Source folder full path 
* {@code args[1]} Target folder full path
* {@code args[2]} Time interval between file/folder synchronization. If doesn't specified or set to 0, the sync will run just once * * * */ public static void main(String[] args) throws Exception { /* fs.src_folder = "test\\src"; fs.target_folder = "test\\target"; fs.time_interval = "5000";*/ FolderSync fs = new FolderSync(); fs = fs.validateArguments(args); fs.run(); } }

FileInfo

/**   
 * This class holds File's information and its properties. 
 **/


public class FileInfo {

	private String fileName = null;
	private File file=null;

	public FileInfo(String file_Name) {
		fileName = file_Name;
		file= new File(file_Name);
	}
	
	
	public boolean isFile()
	{
		return file.isFile();
	}

/**
 * 	
 * Returns file checksum value. Good for verifying integrity of file after
 * transmission.
 * 
 * @return long
 * @throws Exception
 */
	public long calcChecksum() throws Exception {
		long checksum = 0;
		try {

			CheckedInputStream cis = null;
			try {
				// Compute CRC32 checksum
				cis = new CheckedInputStream(
						new FileInputStream(this.fileName), new CRC32());

			} catch (FileNotFoundException e) {
				System.err.println(this.fileName+"  doesn't exists or not accessible");
			e.printStackTrace();
		
			}

			byte[] buf = new byte[128];
			while (cis.read(buf) >= 0) {
			}

			checksum = cis.getChecksum().getValue();

		    cis.close();
		} catch (IOException e) {
			e.printStackTrace();
			throw new Exception("File IO Error");
		}
		
		
		return checksum;
	}

	/**
	 * 
	 * Returns file MD5sum value. Time consuming to calculate, good for critical files 
	 * 
	 * @return String
	 * @throws Exception
	 */
	
	public String calcMD5sum()  throws NoSuchAlgorithmException, FileNotFoundException  {
		MessageDigest digest = MessageDigest.getInstance("MD5");
		File f = new File(this.fileName);
		InputStream is = new FileInputStream(f);
		String output=null;					
		byte[] buffer = new byte[8192];
		int read = 0;
		try {
			while( (read = is.read(buffer)) > 0) {
				digest.update(buffer, 0, read);
			}		
			byte[] md5sum = digest.digest();
			BigInteger bigInt = new BigInteger(1, md5sum);
			output = bigInt.toString(16);
		}
		catch(IOException e) {
			throw new RuntimeException("Unable to process file for MD5", e);
		}
		finally {
			try {
				is.close();
			}
			catch(IOException e) {
				throw new RuntimeException("Unable to close input stream for MD5 calculation", e);
			}
		}		
		
		return output;
	}

	
	
	
	
	
	
	/**
	 * Returns file's size in bytes.
	 * 
	 */
	public long calcSize() throws Exception {
		long fileSize = 0;

		//File f = new File(this.fileName);

		if (file.exists())
			fileSize = file.length();
		else {
			System.err.println(fileName + " File Not Found");
			throw new Exception(fileName + " File Not Found");
		}

		return fileSize;
	}
	
	
		
		

	/*
	 * Waits until file is minutes_toWait minutes older
	 */

	public void delayFileTimeStamp(File f, float minutes_toWait) {
		double file_age = 0; // The duration it was on server
        
		while (file_age < (minutes_toWait * 60 * 1000))
			file_age = System.currentTimeMillis() - f.lastModified();
          

	}

}

Note: I can see there is lot of room to further optimize the code and make it short. I have had written this piece of code on ad-hoc basis to get this in couple of hour for an urgency.

The FileUtil class and its delete and copy functions are not provided. Please let me know if you were unable to write your own.