NIO SSL Server and HTML5 Secure WebSockets

Hi everyone,

this post relates to the “very old” topic http://www.java-gaming.org/topics/nio-ssl-server/21984/view started by Riven.

I’m trying to implement HTML 5 “Secure WebSocket” (TLS/SSL) connections with my existing Java Server using NIO channels. Up to now I have successfully implemented unsecure “WebSocket” connections.

Javascript example code for unsecure WebSocket connections

var connection = new WebSocket('ws://192.168.166.150:9005');

Javascript example code for secure WebSocket connections

var connection = new WebSocket('wss://192.168.166.150:9005');

Javascript Code is executed in Google Chrome or Firefox.

Here’s my server side JAVA code called on incoming data

private void read(SelectionKey key) throws IOException
	{
		SocketChannel socketChannel = (SocketChannel) key.channel();

		// Clear out our read buffer so it's ready for new data
		this.readBuffer.clear();

		// Attempt to read off the channel
		int numRead;
		try
		{
			numRead = socketChannel.read(this.readBuffer);
		}
		catch(IOException e)
		{
			// The remote forcibly closed the connection, cancel
			// the selection key and close the channel.
			this.checkRemoveClient(key);
			return;
		}

		if(numRead == -1)
		{
			// Remote entity shut the socket down cleanly. Do the
			// same from our end and cancel the channel.
			this.checkRemoveClient(key);
			return;
		}
		else
		{
			// Hand the data off to our clientRequestHandler thread
			Client client = (Client)this.clientList.get(socketChannel.hashCode());
			
			byte[] rawData;
			if(this.isSSLEnabled())
			{
				this.writeLogMessage("PLAIN DATA read ["+(new String(this.readBuffer.array()))+"]");

			    this.readBuffer.flip();

			    ByteBuffer copy = ByteBuffer.allocateDirect(numRead);
			    copy.put(this.readBuffer);
			    copy.flip();

			    ClientRequestHandlerSSL crhSSL = new ClientRequestHandlerSSL(this, client, sslContext.createSSLEngine(), this.readBuffer.capacity());
			    crhSSL.receive(copy);
			}
			else
			{
				rawData = this.readBuffer.array();
				ClientRequestHandler crh = new ClientRequestHandler(this, client, rawData, numRead);
				crh.run();
			}
			//
			//this.clientRequestHandler.processData(this, client, this.readBuffer.array(), numRead);
		}
	}

I used the JAVA code example from Riven and adapted it to my existing code architecture for handling SSL data coming over my socket channel:

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.SocketChannel;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;

import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLEngineResult;
import javax.net.ssl.SSLException;
import javax.net.ssl.SSLSession;

public class ClientRequestHandlerSSL implements Runnable
{
   private ByteBuffer wrapSrc, unwrapSrc;
   private ByteBuffer wrapDst, unwrapDst;

   private SSLEngine  engine;
   private Executor   ioWorker, taskWorkers;
   
   private MyServer serverHandle;
   private SocketChannel socketChannel;
   private Client client;

   /**
    * RECEIVE
    * @param serverHandle
    * @param client
    * @param engine
    * @param bufferSize
    * @param ioWorker
    * @param taskWorkers
    */
   public ClientRequestHandlerSSL(MyServer serverHandle, Client client, SSLEngine engine, int bufferSize)
   {
	   System.out.println("construct read");
	   this.serverHandle = serverHandle;
	   this.socketChannel = null;
	   this.client = client;
   System.out.println("Allocate RCV Buffer size: ["+bufferSize+"]");
	   this.wrapSrc = ByteBuffer.allocateDirect(bufferSize);
	   this.wrapDst = ByteBuffer.allocateDirect(bufferSize);

	   this.unwrapSrc = ByteBuffer.allocateDirect(bufferSize);
	   this.unwrapDst = ByteBuffer.allocateDirect(bufferSize);

	   this.unwrapSrc.limit(0);

	   this.engine = engine;
	   this.engine.setUseClientMode(false);
	   this.engine.setNeedClientAuth(false);
	   this.engine.setWantClientAuth(false);
	   try
	   {
		   this.engine.beginHandshake();
	   }
	   catch(SSLException e) {
		   this.serverHandle.writeLogMessage("Can not begin handshake ["+e.getMessage()+"]");
	   }
	   
	   this.ioWorker = Executors.newSingleThreadExecutor();
	   this.taskWorkers = Executors.newFixedThreadPool(4);

	   this.ioWorker.execute(this);
   }
   
   /**
    * SEND
    * 
    * @param serverHandle
    * @param socketChannel
    * @param engine
    * @param bufferSize
    * @param ioWorker
    * @param taskWorkers
    */
   public ClientRequestHandlerSSL(MyServer serverHandle, SocketChannel socketChannel, SSLEngine engine, int bufferSize)
   {
	   System.out.println("construct send");
	   this.serverHandle = serverHandle;
	   this.socketChannel = socketChannel;
	   this.client = null;
System.out.println("Allocate SND Buffer size: ["+bufferSize+"]");
	   this.wrapSrc = ByteBuffer.allocateDirect(bufferSize);
	   this.wrapDst = ByteBuffer.allocateDirect(bufferSize);

	   this.unwrapSrc = ByteBuffer.allocateDirect(bufferSize);
	   this.unwrapDst = ByteBuffer.allocateDirect(bufferSize);

	   this.unwrapSrc.limit(0);

	   this.engine = engine;
	   this.engine.setUseClientMode(false);
	   this.engine.setNeedClientAuth(false);
	   this.engine.setWantClientAuth(false);
	   
	   try
	   {
		   this.engine.beginHandshake();
	   }
	   catch(SSLException e) {
		   this.serverHandle.writeLogMessage("Can not begin handshake ["+e.getMessage()+"]");
	   }
	   
	   this.ioWorker = Executors.newSingleThreadExecutor();
	   this.taskWorkers = Executors.newFixedThreadPool(4);

	   this.ioWorker.execute(this);
   }

   public void handleSendData(ByteBuffer decrypted)
   {
	   try
	   {
		   this.serverHandle.writeLogMessage("SSL data send ["+(new String(decrypted.array()))+"]");
		   this.socketChannel.write(decrypted);
	   }
	   catch (IOException e)
	   {
		   e.printStackTrace();
	   }
   }

   public void handleReceivedData(ByteBuffer encrypted)
   {
	   byte[] dst = new byte[encrypted.remaining()];
	   encrypted.get(dst);
       

	   this.serverHandle.writeLogMessage("SSL data received ["+(new String(dst))+"]");
	   this.serverHandle.writeLogMessage("=======================================================================================");
       this.serverHandle.writeLogMessage("SSL data received decoded ["+(MessageDeEncoder.decodeWebSocketFrame(dst))+"]");
   }

   public void onHandshakeFailure(Exception cause)
   {
	   System.out.println("handshake failure");

       cause.printStackTrace();
   }

   public void onHandshakeSuccess()
   {
	   System.out.println("handshake success");

       SSLSession session = engine.getSession();

       try
       {
          System.out.println("- local principal: " + session.getLocalPrincipal());
          System.out.println("- remote principal: " + session.getPeerPrincipal());
          System.out.println("- using cipher: " + session.getCipherSuite());
       }
       catch (Exception exc)
       {
          exc.printStackTrace();
       }
   }

   public void onClosed()
   {
	   
   }

   public void send(final ByteBuffer data)
   {
	   //wrapSrc.clear();
	   System.out.println("send");
      this.ioWorker.execute(new Runnable()
      {
         @Override
         public void run()
         {
            wrapSrc.put(data);

            ClientRequestHandlerSSL.this.run();
         }
      });
   }

   public void receive(final ByteBuffer data)
   {
	  unwrapSrc.clear();
	  //unwrapDst.clear();
	   System.out.println("receive");
      this.ioWorker.execute(new Runnable()
      {
         @Override
         public void run()
         {
            unwrapSrc.put(data);

            ClientRequestHandlerSSL.this.run();
         }
      });
   }

   public void run()
   {
      // executes non-blocking tasks on the IO-Worker
System.out.println("RUN");
      while (this.step())
      {
         continue;
      }
      System.out.println("Thread end");
      // apparently we hit a blocking-task...
   }

   private boolean step()
   {
      switch (engine.getHandshakeStatus())
      {
         case NOT_HANDSHAKING:
        	 System.out.println("not handshaking");
            boolean anything = false;
            {
               if (wrapSrc.position() > 0)
                  anything |= this.wrap();
               if (unwrapSrc.position() > 0)
                  anything |= this.unwrap();
            }
            return anything;

         case NEED_WRAP:
        	 System.out.println("need wrap");
            if (!this.wrap())
               return false;
            break;

         case NEED_UNWRAP:
        	 System.out.println("need unwrap");
            if (!this.unwrap())
               return false;
            break;

         case NEED_TASK:
        	 System.out.println("need task");
            final Runnable sslTask = engine.getDelegatedTask();
            Runnable wrappedTask = new Runnable()
            {
               @Override
               public void run()
               {
                  System.out.println("async SSL task: " + sslTask);
                  long t0 = System.nanoTime();
                  sslTask.run();
                  long t1 = System.nanoTime();
                  System.out.println("async SSL task took: " + (t1 - t0) / 1000000 + "ms");

                  // continue handling I/O
                  ioWorker.execute(ClientRequestHandlerSSL.this);
               }
            };
            taskWorkers.execute(wrappedTask);
            return false;

         case FINISHED:
        	 System.out.println("finished");
            throw new IllegalStateException("FINISHED");
      }
      System.out.println("true");
      return true;
   }

   private boolean wrap()
   {
      SSLEngineResult wrapResult;

      try
      {
         wrapSrc.flip();
         wrapResult = engine.wrap(wrapSrc, wrapDst);
         wrapSrc.compact();
      }
      catch (SSLException exc)
      {
         this.onHandshakeFailure(exc);
         return false;
      }

      switch (wrapResult.getStatus())
      {
         case OK:
        	 System.out.println("Wrap: OK");
            if (wrapDst.position() > 0)
            {
               wrapDst.flip();
               System.out.println("calling handleReceiveData");
               this.handleReceivedData(wrapDst);
               wrapDst.compact();
            }
            break;

         case BUFFER_UNDERFLOW:
        	 System.out.println("Wrap: Buffer underflow");
            // try again later
            break;

         case BUFFER_OVERFLOW:
        	 System.out.println("Wrap: Buffer overflow");
            throw new IllegalStateException("failed to wrap");

         case CLOSED:
            this.onClosed();
            return false;
      }

      return true;
   }

   private boolean unwrap()
   {
      SSLEngineResult unwrapResult;

      try
      {
         unwrapSrc.flip();
         unwrapResult = engine.unwrap(unwrapSrc, unwrapDst);
         unwrapSrc.compact();
      }
      catch (SSLException exc)
      {
         this.onHandshakeFailure(exc);
         return false;
      }

      switch (unwrapResult.getStatus())
      {
         case OK:
        	 System.out.println("Unwrap: OK");
            if (unwrapDst.position() > 0)
            {
               unwrapDst.flip();
               System.out.println("calling handleSendData");
               this.handleSendData(unwrapDst);
               unwrapDst.compact();
            }
            break;

         case CLOSED:
            this.onClosed();
            return false;

         case BUFFER_OVERFLOW:
            throw new IllegalStateException("failed to unwrap");

         case BUFFER_UNDERFLOW:
        	 System.out.println("Unwrap: Buffer underflow");
            return false;
      }

      switch (unwrapResult.getHandshakeStatus())
      {
         case FINISHED:
            this.onHandshakeSuccess();
            return false;
      }

      return true;
   }
}

My SSLContext implementation

KeyStore ks = KeyStore.getInstance("JKS");
				KeyStore ts = KeyStore.getInstance("JKS");
				FileInputStream fin = new FileInputStream(keyStoreFilePath);
				ks.load(fin, keyStoreFilePassword.toCharArray());
		
				// Set up key manager factory to use our key store
				// Assume key password is the same as the key store file
				// password
				KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
				kmf.init(ks, keyStoreFilePassword.toCharArray());
				
				TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
				tmf.init(ts);
				
				// Initialize the SSLContext to work with our key managers.
				sslContext = SSLContext.getInstance("TLS");
				sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);

This is the result printed by ClientRequestHandlerSSL class to eclipse console when I open the secure websocket in Chrome:
construct read
Allocate RCV Buffer size: [65536]
receive
RUN
need unwrap
Unwrap: Buffer underflow
Thread end
RUN
need unwrap
Unwrap: OK
true
need task
Thread end
async SSL task: sun.security.ssl.Handshaker$DelegatedTask@1d0fe80
async SSL task took: 44ms
RUN
need wrap
Wrap: OK
calling handleReceiveData
true
need unwrap
Unwrap: Buffer underflow
Thread end

The code seems to work as but the result data written to my server logfile by “handleReceivedData” seems to be messed up. I can partially see data of my SSL certificate along with unreadable data. Obviously I do something terribly wrong.
I hope someone can please point me in the right direction? :slight_smile: