Mediaflux Plugin Development: Create dataset from HTTP URL (with Basic Auth)

The example shows how to create a dataset asset from a password protected URL (Basic HTTP Auth).

	package example;
	import java.io.File;
	import java.io.InputStream;
	import java.net.HttpURLConnection;
	import java.net.URI;
	import java.net.URL;
	import java.util.Base64;
	import arc.mf.plugin.PluginService;
	import arc.mf.plugin.ServiceExecutor;
	import arc.streams.StreamCopy;
	import arc.xml.XmlDocMaker;
	import arc.xml.XmlWriter;
	public class CreateDatasetFromURL {
	public static interface ResponseInputStreamHandler {
	void handleResponseInputStream(InputStream in) throws Throwable;
	}
	/**
	* Do HTTP GET for the specified URL (and user credentials).
	*
	* @param uri
	* @param username
	* @param password
	* @param rh
	* @throws Throwable
	*/
	public static void httpGet(String uri, String username, String password, final ResponseInputStreamHandler rh)
	throws Throwable {
	URL url = URI.create(uri).toURL();
	HttpURLConnection conn = (HttpURLConnection) url.openConnection();
	try {
	conn.setDoOutput(false);
	conn.setDoInput(true);
	conn.setRequestMethod("GET");
	String authorization = "Basic "
	+ Base64.getEncoder().encodeToString((username + ":" + password).getBytes());
	conn.setRequestProperty("Authorization", authorization);
	int responseCode = conn.getResponseCode();
	String responseMessage = conn.getResponseMessage();
	InputStream responseInputStream = null;
	try {
	if (responseCode != HttpURLConnection.HTTP_OK) {
	throw new Exception("Unexpected HTTP response: " + responseCode + " " + responseMessage);
	}
	responseInputStream = conn.getInputStream();
	rh.handleResponseInputStream(responseInputStream);
	} finally {
	if (responseInputStream != null) {
	responseInputStream.close();
	}
	}
	} finally {
	conn.disconnect();
	}
	}
	/**
	* Download from the HTTP URL and save to the specified file.
	*
	* @param httpUrl
	* @param httpUsername
	* @param httpPassword
	* @param dstFile
	* @throws Throwable
	*/
	public static void download(String httpUrl, String httpUsername, String httpPassword, final File dstFile)
	throws Throwable {
	httpGet(httpUrl, httpUsername, httpPassword, new ResponseInputStreamHandler() {
	@Override
	public void handleResponseInputStream(InputStream in) throws Throwable {
	StreamCopy.copy(in, dstFile);
	}
	});
	}
	/**
	* Create a dataset from HTTP URL (and basic http auth).
	*
	* @param cxn
	* @param studyCid
	* @param httpUrl
	* @param httpUsername
	* @param httpPassword
	* @throws Throwable
	*/
	public static void createDataset(final ServiceExecutor executor, final String studyCid, final String httpUrl,
	String httpUsername, String httpPassword, final XmlWriter w) throws Throwable {
	httpGet(httpUrl, httpUsername, httpPassword, new ResponseInputStreamHandler() {
	@Override
	public void handleResponseInputStream(InputStream in) throws Throwable {
	XmlDocMaker dm = new XmlDocMaker("args");
	dm.add("pid", studyCid);
	// TODO: add other metadata if needed
	String fileName = extractFileNameFromUrl(httpUrl);
	String fileExt = extractFileExtension(fileName);
	String fileMimeType = mimeTypeFromExt(fileExt);
	PluginService.Input input = new PluginService.Input(in, -1, fileMimeType, fileName);
	executor.execute("om.pssd.dataset.derivation.create", dm.root(), new PluginService.Inputs(input), null);
	}
	});
	}
	private static String extractFileExtension(String fileName) {
	if (fileName != null) {
	int idx = fileName.lastIndexOf('.');
	if (idx > 0) {
	return fileName.substring(idx + 1);
	}
	}
	return null;
	}
	private static String extractFileNameFromUrl(String url) {
	if (url != null) {
	int idx = url.lastIndexOf('/');
	if (idx > 0) {
	return url.substring(idx + 1);
	}
	}
	return null;
	}
	private static String mimeTypeFromExt(String ext) {
	if (ext != null) {
	if (ext.equalsIgnoreCase("gz")) {
	return "application/x-gzip";
	}
	if (ext.equalsIgnoreCase("zip")) {
	return "application/zip";
	}
	// The above just example.
	// You add other know mime types here.
	}
	return null;
	}
	}

view raw CreateDatasetFromUrl.java hosted with ❤ by GitHub

Java: HTTP GET data from URL (with basic HTTP Auth)

The example shows how to do HTTP GET to download a file from URL. It also include an example to call Mediaflux client api to create a dataset from the url by piping the streams.

	package example;
	import java.io.File;
	import java.io.InputStream;
	import java.io.OutputStream;
	import java.io.PipedInputStream;
	import java.io.PipedOutputStream;
	import java.net.HttpURLConnection;
	import java.net.URI;
	import java.net.URL;
	import java.util.Base64;
	import arc.mf.client.ServerClient;
	import arc.mf.client.archive.Archive;
	import arc.streams.StreamCopy;
	import arc.streams.StreamCopy.AbortCheck;
	import arc.xml.XmlStringWriter;
	public class ImportDataByURL {
	public static void main(String[] args) throws Throwable {
	String url = "https://yourorg.org/path/to/file.gz";
	String username = "user";
	String password = "password";
	File of = new File("/local/path/to/file.gz");
	httpGet(url, username, password, new ResponseInputStreamHandler() {
	@Override
	public void handleResponseInputStream(InputStream in) throws Throwable {
	StreamCopy.copy(in, of);
	}
	});
	}
	public static interface ResponseInputStreamHandler {
	void handleResponseInputStream(InputStream in) throws Throwable;
	}
	/**
	* Do HTTP GET for the specified URL (and user credentials).
	*
	* @param uri
	* @param username
	* @param password
	* @param rh
	* @throws Throwable
	*/
	public static void httpGet(String uri, String username, String password, final ResponseInputStreamHandler rh)
	throws Throwable {
	URL url = URI.create(uri).toURL();
	HttpURLConnection conn = (HttpURLConnection) url.openConnection();
	try {
	conn.setDoOutput(false);
	conn.setDoInput(true);
	conn.setRequestMethod("GET");
	String authorization = "Basic "
	+ Base64.getEncoder().encodeToString((username + ":" + password).getBytes());
	conn.setRequestProperty("Authorization", authorization);
	int responseCode = conn.getResponseCode();
	String responseMessage = conn.getResponseMessage();
	InputStream responseInputStream = null;
	try {
	if (responseCode != HttpURLConnection.HTTP_OK) {
	throw new Exception("Unexpected HTTP response: " + responseCode + " " + responseMessage);
	}
	responseInputStream = conn.getInputStream();
	rh.handleResponseInputStream(responseInputStream);
	} finally {
	if (responseInputStream != null) {
	responseInputStream.close();
	}
	}
	} finally {
	conn.disconnect();
	}
	}
	/**
	* Download from the HTTP URL and save to the specified file.
	*
	* @param httpUrl
	* @param httpUsername
	* @param httpPassword
	* @param dstFile
	* @throws Throwable
	*/
	public static void download(String httpUrl, String httpUsername, String httpPassword, File dstFile)
	throws Throwable {
	httpGet(httpUrl, httpUsername, httpPassword, new ResponseInputStreamHandler() {
	@Override
	public void handleResponseInputStream(InputStream in) throws Throwable {
	StreamCopy.copy(in, dstFile);
	}
	});
	}
	/**
	* Create a dataset from HTTP URL (and basic http auth).
	*
	* @param cxn
	* @param studyCid
	* @param httpUrl
	* @param httpUsername
	* @param httpPassword
	* @throws Throwable
	*/
	public static void createDataset(ServerClient.Connection cxn, String studyCid, String httpUrl, String httpUsername,
	String httpPassword) throws Throwable {
	XmlStringWriter w = new XmlStringWriter();
	w.add("pid", studyCid);
	// TODO: add other metadata if known
	String fileName = extractFileNameFromUrl(httpUrl);
	String fileExt = extractFileExtension(fileName);
	String fileMimeType = mimeTypeFromExt(fileExt);
	Archive.declareSupportForAllTypes();
	ServerClient.Input sci = new ServerClient.GeneratedInput(fileMimeType, fileExt, fileName, -1, null) {
	@Override
	protected void copyTo(OutputStream os, AbortCheck ac) throws Throwable {
	PipedInputStream pis = new PipedInputStream();
	PipedOutputStream pos = new PipedOutputStream(pis);
	Throwable[] workerException = new Throwable[1];
	Thread workerThread = new Thread(new Runnable() {
	@Override
	public void run() {
	try {
	httpGet(httpUrl, httpUsername, httpPassword, new ResponseInputStreamHandler() {
	@Override
	public void handleResponseInputStream(InputStream in) throws Throwable {
	try {
	StreamCopy.copy(in, pos);
	} finally {
	pos.close();
	}
	}
	});
	} catch (Throwable e) {
	System.err.println("Error in worker thread: " + Thread.currentThread().getName());
	e.printStackTrace(System.err);
	workerException[0] = e;
	}
	}
	}, "HttpGet");
	try {
	workerThread.start();
	StreamCopy.copy(pis, os);
	workerThread.join();
	if (workerException[0] != null) {
	throw new Exception("Worker thread exception: " + workerException[0].getMessage(),
	workerException[0]);
	}
	} finally {
	os.close();
	}
	}
	};
	cxn.execute("om.pssd.dataset.derivation.create", w.document(), sci);
	}
	private static String extractFileExtension(String fileName) {
	if (fileName != null) {
	int idx = fileName.lastIndexOf('.');
	if (idx > 0) {
	return fileName.substring(idx + 1);
	}
	}
	return null;
	}
	private static String extractFileNameFromUrl(String url) {
	if (url != null) {
	int idx = url.lastIndexOf('/');
	if (idx > 0) {
	return url.substring(idx + 1);
	}
	}
	return null;
	}
	private static String mimeTypeFromExt(String ext) {
	if (ext != null) {
	if (ext.equalsIgnoreCase("gz")) {
	return "application/x-gzip";
	}
	if (ext.equalsIgnoreCase("zip")) {
	return "application/zip";
	}
	// The above just example.
	// You add other know mime types here.
	}
	return null;
	}
	}

view raw ImportDataFromURL.java hosted with ❤ by GitHub

encode Base64 string in command line

• Encode base64 string:

  echo -n Secret | base64
  

• Decode base64 string:

  echo -n U2VjcmV0 | base64 -D
  

Note:

When encoding, -n must be specified. otherwise, a newline character will always be appended, which will cause unexpected result.

See http://askubuntu.com/questions/694216/base64-encode-is-giving-ambigious-results

Maven: skip unit tests

<properties>
    <maven.test.skip>true</maven.test.skip>
</properties>

make Java HttpURLConnection support MKCOL PROFIND and other non-standard HTTP methods

    private static void setRequestMethod(HttpURLConnection conn, String method) throws Throwable {
        try {
            conn.setRequestMethod(method);
        } catch (ProtocolException e) {
            Class<?> c = conn.getClass();
            Field methodField = null;
            Field delegateField = null;
            try {
                delegateField = c.getDeclaredField("delegate");
            } catch (NoSuchFieldException nsfe) {

            }
            while (c != null && methodField == null) {
                try {
                    methodField = c.getDeclaredField("method");
                } catch (NoSuchFieldException nsfe) {

                }
                if (methodField == null) {
                    c = c.getSuperclass();
                }
            }
            if (methodField != null) {
                methodField.setAccessible(true);
                methodField.set(conn, method);
            }

            if (delegateField != null) {
                delegateField.setAccessible(true);
                HttpURLConnection delegate = (HttpURLConnection) delegateField.get(conn);
                setRequestMethod(delegate, method);
            }
        }
    }