Two utility methods to validate the downloaded file

Currently, we validate the file size only. We could validate the content using any hashing function later.
aalhossary · Jan 26, 2023 · Feb 20, 2022 · Feb 21, 2022 · Feb 22, 2022 · Feb 22, 2022
commit b8e69a4b022aaa6b5f5ecfbd31de70e32358ce85
diff --git a/biojava-core/src/main/java/org/biojava/nbio/core/util/FileDownloadUtils.java b/biojava-core/src/main/java/org/biojava/nbio/core/util/FileDownloadUtils.java
@@ -23,9 +23,11 @@
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.PrintStream;
 import java.net.HttpURLConnection;
 import java.net.SocketTimeoutException;
 import java.net.URL;
@@ -39,6 +41,7 @@
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Scanner;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -161,6 +164,64 @@ public static void downloadFile(URL url, File destination) throws IOException {
 		tempFile.delete();
 
 	}
+
+	public static void createValidationFiles(URL url, File localDestination, URL hashURL){
+		try {
+			URLConnection resourceConnection = url.openConnection();
+			createValidationFiles(resourceConnection, localDestination, hashURL);
+		} catch (IOException e) {
+			logger.warn("could not open connection to resource file due to exception", e);
+		}
+	}
+	public static void createValidationFiles(URLConnection resourceUrlConnection, File localDestination, URL hashURL){
+		long size = resourceUrlConnection.getContentLengthLong();
+		if(size != -1) {
+			System.out.println("Content-Length: " + size);
+			File sizeFile = new File(localDestination.getParentFile(), localDestination.getName()+".size");
+			try (PrintStream sizePrintStream = new PrintStream(sizeFile)) {
+				sizePrintStream.print(size);
+				sizePrintStream.close();
+			} catch (FileNotFoundException e) {
+				logger.warn("could not write validation size file due to exception", e);
+			}
+		}
+
+		if(hashURL == null)
+			return;
+
+		try {
+			File hashFile = new File(localDestination.getParentFile(), localDestination.getName()+".hash");
+			downloadFile(hashURL, hashFile);
+		} catch (IOException e) {
+			logger.warn("could not write validation hash file due to exception", e);
+		}
+	}
+
+	public static boolean validateFile(File localFile) {
+		File sizeFile = new File(localFile.getParentFile(), localFile.getName()+".size");
+		if(sizeFile.exists()) {
+			Scanner scanner = null;
+			try {
+				scanner = new Scanner(sizeFile);
+				long expectedSize = scanner.nextLong();
+				if (expectedSize != localFile.length()) {
+					logger.warn("File size does not match expected");
+					return false;
+				}
+			} catch (FileNotFoundException e) {
+				logger.warn("could not validate size of file ["+ localFile+ "] due to exception", e);
+			} finally {
+				scanner.close();
+			}
+		}
+
+		File hashFile = new File(localFile.getParentFile(), localFile.getName()+".hash");
+		if(hashFile.exists()) {
+			throw new UnsupportedOperationException("Not yet implemented");
+		}
+
+		return true;
+	}
 
 	/**
 	 * Converts path to Unix convention and adds a terminating slash if it was

diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/ecod/EcodFactory.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/ecod/EcodFactory.java
@@ -89,6 +89,9 @@ public static EcodDatabase getEcodDatabase(String version) {
 					}
 				} catch (IOException e) {
 					// For parsing errors, just use the requested version
+					// What about corrupted downloading errors?? Amr
+					logger.warn("Cound not get Ecod version, or file is corrupted", e);
+					return null;
 				}
 			}
 			logger.trace("Releasing EcodFactory lock after getting version "+version);

diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/ecod/EcodInstallation.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/ecod/EcodInstallation.java
@@ -369,7 +369,7 @@ private boolean domainsAvailable() {
 		try {
 			File f = getDomainFile();
 
-			if (!f.exists() || f.length() <= 0 )
+			if (! (f.exists() && FileDownloadUtils.validateFile(f)))
 				return false;
 
 			// Re-download old copies of "latest"
@@ -406,7 +406,10 @@ private void downloadDomains() throws IOException {
 			File localFile = getDomainFile();
 
 			logger.info("Downloading {} to: {}",domainsURL, localFile);
+			FileDownloadUtils.createValidationFiles(domainsURL, localFile, null);
 			FileDownloadUtils.downloadFile(domainsURL, localFile);
+			if(! FileDownloadUtils.validateFile(localFile))
+				throw new IOException("Downloaded file invalid: "+ localFile);
 		} catch (MalformedURLException e) {
 			logger.error("Malformed url: "+ url + DOMAINS_PATH + getDomainFilename(),e);
 		} finally {

diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/LocalPDBDirectory.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/LocalPDBDirectory.java
@@ -373,6 +373,9 @@ protected InputStream getInputStream(PdbId pdbId) throws IOException{
 			throw new IOException("Structure "+pdbId+" not found and unable to download.");
 		}
 
+		if(! FileDownloadUtils.validateFile(file))
+			throw new IOException("Downloaded file invalid: "+file);
+
 		InputStreamProvider isp = new InputStreamProvider();
 
 		InputStream inputStream = isp.getInputStream(file);
@@ -395,6 +398,8 @@ public void prefetchStructure(String pdbId) throws IOException {
 		if(!file.exists()) {
 			throw new IOException("Structure "+pdbId+" not found and unable to download.");
 		}
+		if(! FileDownloadUtils.validateFile(file))
+			throw new IOException("Downloaded file invalid: "+file);
 	}
 
 	/**
@@ -576,7 +581,10 @@ private File downloadStructure(PdbId pdbId, String pathOnServer, boolean obsolet
 		logger.info("Fetching " + ftp);
 		logger.info("Writing to "+ realFile);
 
+		FileDownloadUtils.createValidationFiles(url, realFile, null);
 		FileDownloadUtils.downloadFile(url, realFile);
+		if(! FileDownloadUtils.validateFile(realFile))
+			throw new IOException("Downloaded file invalid: "+realFile);
 
 		return realFile;
 	}

diff --git a/...ava-structure/src/main/java/org/biojava/nbio/structure/io/sifts/SiftsMappingProvider.java b/...ava-structure/src/main/java/org/biojava/nbio/structure/io/sifts/SiftsMappingProvider.java
@@ -87,10 +87,15 @@ public static List<SiftsEntity> getSiftsMapping(String pdbId) throws IOException
 		if ( ! dest.exists()){
 			String u = String.format(fileLoc,pdbId);
 			URL url = new URL(u);
+			logger.debug("Downloading SIFTS file {} validation metadata.",url);
+			FileDownloadUtils.createValidationFiles(url, dest, null);
 			logger.debug("Downloading SIFTS file {} to {}",url,dest);
 			FileDownloadUtils.downloadFile(url, dest);
 		}
 
+		if(! FileDownloadUtils.validateFile(dest))
+			throw new IOException("Downloaded file invalid: "+dest);
+
 		InputStreamProvider prov = new InputStreamProvider();
 		InputStream is = prov.getInputStream(dest);
 		SiftsXMLParser parser = new SiftsXMLParser();

diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/scop/ScopInstallation.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/scop/ScopInstallation.java
@@ -741,38 +741,41 @@ protected void downloadComFile() throws FileNotFoundException, IOException{
 
 	protected void downloadFileFromRemote(URL remoteURL, File localFile) throws IOException{
 		logger.info("Downloading " + remoteURL + " to: " + localFile);
+		FileDownloadUtils.createValidationFiles(remoteURL, localFile, null);
 		FileDownloadUtils.downloadFile(remoteURL, localFile);
+		if(! FileDownloadUtils.validateFile(localFile))
+			throw new IOException("Downloaded file invalid: "+localFile);
 	}
 
 	private boolean claFileAvailable(){
 		String fileName = getClaFilename();
 
 		File f = new File(fileName);
 
-		return f.exists() && f.length()>0;
+		return f.exists() && FileDownloadUtils.validateFile(f);
 	}
 
 	private boolean desFileAvailable(){
 		String fileName = getDesFilename();
 
 		File f = new File(fileName);
-		return f.exists() && f.length()>0;
+		return f.exists() && FileDownloadUtils.validateFile(f);
 	}
 
 	private boolean hieFileAvailable(){
 		String fileName = getHieFilename();
 
 		File f = new File(fileName);
 
-		return f.exists() && f.length()>0;
+		return f.exists() && FileDownloadUtils.validateFile(f);
 	}
 
 	private boolean comFileAvailable(){
 		String fileName = getComFilename();
 
 		File f = new File(fileName);
 
-		return f.exists() && f.length()>0;
+		return f.exists() && FileDownloadUtils.validateFile(f);
 	}
 
 	protected String getClaFilename(){