HTTP Range header and Download managers

Prerequisite

Java RandomAccessFile Class

Overview

This article details the uses of HTTP Range header and its use in Download managers.

HTTP Range Header

HTTP Range is a available in HTTP/1.1 and allow to specify the range of bytes. It allow you to specify the starting byte and ending byte of the requested stream.
Range header allow many type of ranges we have used only byte range. Here is the syntax of byte range "Range: bytes=1-100", here 0 is starting byte and 100 is ending byte if ending byte is missing (Range: bytes=1-) it will be taken as the last byte in the file(EOF).
For more information on Range header click here

Range header used in Download manager

• Continuing the download: If you are downloading and download break in the middle because of some external or internal reason, without range header you have to start from the beginning range header allow you to specify the starting byte of the download.
  
• Multi-threading download: you can divide the download of large file into multiple chunks and download them in parallel.
  Below is the code which divide the file in chunks and download in parallel, this program after configuring properly gives you 50% better performance then Orbit download manager.
  

package www.directi.com.junk;

import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;

/*
* this class downloads a file from from given URL in multiple parts. It uses http range header
* $author = Paras Malik(masterofmasters22@gmail.com)
* $date = june 25, 2011
* */

public class DownloadInChunks implements Runnable {
    private static String fileURL = "sourceFileLink"; //file to be downloaded
    private static final String targetFileName = "targetFileName";
    private static final int MAX_BUFFER_SIZE = 1024 * 64 * 4; //maximum number of bytes for which connection will be probed
    private static int chunkOfFile = 1024 * 1024;  //size of one chunk of file to be downloaded
    private int downloadStartingByte;
    private int downloadEndingByte;
    static int fs;

    public DownloadInChunks(int downloadStartingByte, int downloadEndingByte) {
        this.downloadStartingByte = downloadStartingByte;
        this.downloadEndingByte = downloadEndingByte;
    }

    public static void main(String s[]) throws IOException {
        System.out.println(new Date());
        int downloadStartingByte = 0;
        int downloadEndingByte = chunkOfFile;
        int fileSize = getFileSize();
        fs = fileSize;
        while (fileSize > downloadStartingByte) {
            if (downloadEndingByte == fileSize)
                new Thread(new DownloadInChunks(downloadStartingByte, -1)).start();
            else
                new Thread(new DownloadInChunks(downloadStartingByte, downloadEndingByte)).start();
            downloadStartingByte = downloadEndingByte;
            if (downloadEndingByte + chunkOfFile <= fileSize)
                downloadEndingByte = downloadEndingByte + chunkOfFile;
            else
                downloadEndingByte = fileSize;
        }
    }

    static int getFileSize() throws IOException {
        URL url = new URL(fileURL);
        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
        connection.connect();
        if (connection.getResponseCode() != 200) {
            System.out.println("Error return code != 200");
        }
        int contentLength = connection.getContentLength();
        connection.disconnect();
        return contentLength;
    }

    public void downloadPartOfFile(int startingByte, String endingByte) throws IOException {
        URL url = new URL(fileURL);
        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
        int downloaded = 0;
        connection.setRequestProperty("Range", "bytes=" + startingByte + "-" + endingByte); //is endingByte is empty string http request all the bytes till EOF from given starting byte.
        connection.connect();
        if (connection.getResponseCode() != 206) {   // http return 206 as signal of OK if request is send with range header
            System.out.println("Error return code != 206");
        }
        int contentLength = connection.getContentLength();
        RandomAccessFile file = new RandomAccessFile(targetFileName, "rw");
        file.seek(startingByte);
        InputStream stream = connection.getInputStream();
        while (true) {
            byte buffer[];
            if (contentLength - downloaded > MAX_BUFFER_SIZE) {
                buffer = new byte[MAX_BUFFER_SIZE];
            } else {
                buffer = new byte[contentLength - downloaded];
            }
            int read = stream.read(buffer);
            if (read == -1 || downloaded == contentLength)
                break;
            file.write(buffer, 0, read);
            downloaded += read;
        }
        file.close();
    }

    @Override
    public void run() {
        try {
            if (downloadEndingByte == -1)
                downloadPartOfFile(downloadStartingByte, "");
            else
                downloadPartOfFile(downloadStartingByte, String.valueOf(downloadEndingByte));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

RandomAccessFile Read/Write Concurrency

Prerequisite

• Java Basics I/O

Overview

This article details some useful features of RandomAccessFile java class.

RandomAccessFile

Instances of this class support both reading and writing to a random access file. A random access file behaves like a large array of bytes stored in the file system. There is a kind of cursor, or index into the implied array, called the file pointer; input operations read bytes starting at the file pointer and advance the file pointer past the bytes read.
If the random access file is created in read/write mode, then output operations are also available; output operations write bytes starting at the file pointer and advance the file pointer past the bytes written. Output operations that write past the current end of the implied array cause the array to be extended. The file pointer can be set by the seek method.

seek operation
Sets the file-pointer offset, measured from the beginning of this file, at which the next read or write occurs. The offset can be set beyond the end of the file. Setting the offset beyond the end of the file does not change the file length. The file length will change only by writing after the offset has been set beyond the end of the file

Concurrent Read/Write

This is a useful but very dangerous feature. It goes like this "if you create different instances of RandomAccessFile for a same file you can concurrently write to the different parts of the file."
You can create multiple threads pointing to different parts of the file using seek method and multiple threads can update the file at the same time. Seek allow you to move to any part of the file even if it doesn't exist( after EOF ), hence you can move to any location in the newly created file and write bytes on that location. You can open multiple instances of the same file and seek to different locations and write to multiple locations at the same time. Below is a working example to copy one file to another using multiple threads. You can tune this code to perform better then windows .
Note: writing concurrently is only a performance hack. If not properly implemented this can lead to race condition and many other critical bugs. You should avoid doing this unless necessary.

package www.directi.com.junk;

import java.io.IOException;
import java.io.RandomAccessFile;

/*
 * copy file(filePath) to (targetFileName) in chunks. It divide the file in parts of chuckOfFile and copy each part concurrently
 * $author = Paras Malik(masterofmasters22@gmail.com)
 * $Date = 20 June, 2011
*/
public class CopyFile implements Runnable {

    public static String filePath = "source.mkv";  //file to copy
    public static String targetFileName = "target.mkv";
    private static final int MAX_BUFFER_SIZE = 1024 * 1024 * 5;
    private int startingByte;
    private int endingByte;
    private static int chunkOfFile = 1024 * 1024 * 70;  //size of one chunk of file

    public CopyFile(int startingByte, int endingByte) {
        this.startingByte = startingByte;
        this.endingByte = endingByte;
    }

    public static void main(String s[]) throws IOException {
        int copyStartingByte = 0;
        int copyEndingByte = chunkOfFile;
        int fileSize = (int) getFileSize();
        while (fileSize > copyStartingByte) {
            new Thread(new CopyFile(copyStartingByte, copyEndingByte)).start();
            copyStartingByte = copyEndingByte;
            if (copyEndingByte + chunkOfFile <= fileSize)
                copyEndingByte = copyEndingByte + chunkOfFile;
            else
                copyEndingByte = fileSize;
        }
    }

    static long getFileSize() throws IOException {
        RandomAccessFile f = new RandomAccessFile(filePath, "r");
        long contentLength = f.length();
        return contentLength;
    }

    /* copy one part of the file starting from startingByte till endingByte*/
    public void copyPartOfFile(int startingByte, int endingByte) throws IOException {
        int copied = 0;
        int contentLength = endingByte - startingByte;
        RandomAccessFile sourceFile = new RandomAccessFile(filePath, "rw");
        RandomAccessFile targetFile = new RandomAccessFile(this.targetFileName, "rw");
        targetFile.seek(startingByte);
        sourceFile.seek(startingByte);
        while (true) {
            byte buffer[];
            if (contentLength - copied > MAX_BUFFER_SIZE) {
                buffer = new byte[MAX_BUFFER_SIZE];
            } else {
                buffer = new byte[contentLength - copied];
            }
            int read = sourceFile.read(buffer, 0, buffer.length);

            if (read == -1 || copied == contentLength)
                break;
            targetFile.write(buffer, 0, read);
            copied += read;
        }
        sourceFile.close();
        targetFile.close();
    }

    @Override
    public void run() {
        try {
            copyPartOfFile(startingByte, endingByte);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}