Music/Sound

The resources thread is getting a bit long so I thought I would start one with more dedicated focus.

Over the last few weeks I have been working on a programmable sound generator and this is where I am at so far: Synth4K
The 7-bit instructions for the generator are encoded in utf-8 strings. Sound effects use around 6 bytes each, a full song can be 1K (uncompressed). The sound generator itself is around 1K although I think I can shrink that considerably if I get a chance to work on it some more (really need to start on a game).

I have a few sound effects and a couple of songs (Mario/Zelda) in the demo but will add some more generic ones later on.

This is great!

I have never really been into the music/ sound gen of games. Hopefully using/abusing :slight_smile: this resource I will be able to add much needed sounds to my entry.

I dont suppose you could add comments to the code to help those of us (namely me) who are total noobes to java sound api and sound generation in general?

I’m not much into sound either. But here’s some food for thought: Markov Chains. Don’t use training data, just some small hardcoded data tables and perhaps bop between different chain orders. This could be coupled with some small grammar.

It stuttered badly on my machine until I reduced the sample rate to a quarter to the value.

But it’s impressive. I’m not sure whether I’m doing a 4k this year, but I wanted to have sound. So this is nice.

Another example showing how to synthesize a tune:


/*
 * S.java
 *
 * Created on 14 April 2009, 14:34
 */

/**
 *
 * @author  Alan Waddington
 */
import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.AudioFormat;
import javax.sound.sampled.DataLine;
import javax.sound.sampled.Clip;

import java.awt.Graphics;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferInt;
import javax.swing.JFrame;

public final class S extends JFrame {
    private final static int   SCREENHEIGHT   = 480;    // Screen height
    private final static int   SCREENWIDTH    = 640;    // Screen width

    private final static float RATE           = 16000f;// Music sample rate
    private final static float AMPLITUDE      = 4000f; // Music amplitude

    private byte musicLoop[];           // Array of 16 bit music sound samples
    private Clip music;                 // Clip to play the above music track
    
int maxNote = 261;
int length = 512;
int tempo = 200;
String notes = "2gK8 I9k&+.y{^10[.p0l\"V^#pvoM$+#N8 I9k&+.y{^1%%bo0h2#^#Y4L8 \\4L8 \\4L8 Y4L8 f1M&$}xV^#g\"V^#j\"V^#QTd^#57@^#g\"V^#*55N$z=Y_1O(// aVU^#Nd1^#1Gl]#Pau7*g\"V^#l\"V^#Qt]]#DTd^#g\"V^#l\"V^#&oj%$M.A^#NTd^#57@^#Nd1^#1Gl]#Jd1^#1Gl]#,Gl]#77@^#57@^#3i481}xV^#l\"V^#Qt]]#KTd^#g\"V^#l\"V^#27@^#h2#^#c2#^#h2#^#Jt]]#6tx%$o+M8 V4Y_ o+M8 V4Y_ d[2^#1Gl]#e2#^#Jt]]#?i3t$Fw%^#8-0&$`k^]#,W9]#k)H]#ZDx7 <T}6*r_/&$v'}6*r_/&$svP7*h2#^#qxz]#1Gl]#@DQ7*h2#^#h2#^#c2#^#Jt]]#p)H]#F%+]#+/U%$P}T7 :'T7 4'a^ P}T7 :'T7 :'T7 :'T7 :'T7 :'T7 F%+]#A%+]#{W*]#uVT%$lKF7 4'a^ f+g'!)#N8 Jt]]#,W9]#MyDL$V2<6*{W*]#PR7%$*e;6*{W*]#{W*]#{W*]#{W*]#{W*]#_*9]#n9t\\#:'T7 VTE7 4'a^ lKF7 4'a^ f+g'!)#N8 Jt]]#,W9]#%U   ";

/** Creates a new instance of S */
    public S() {
        // Create array of music frequencies
        double[] freq = new double[100];
        double f = 16.351562;        // Frequency of C0 (C4 is middle C)
        for (int i=0; i<7*12; i++) {
            freq[i] = f;
            f *= 1.0594630944;       // Evenly tempered scale (12th root 2)
        }        

        musicLoop = new byte[(int)( length*30f/tempo*RATE )];

        // Read each note and render it into the loop
        int semiQuaver = 0;             // Start time for next note
        int byte0, byte1, twoNotes=0;
        for (int i=0; i<maxNote; i++) {
            if ((i & 1)==0) {
                // Decode 5 chars to 1 int
                twoNotes = 0;
                for (int j=4;j>=0;j--)
                    twoNotes = twoNotes*95+(int)notes.charAt(5*(i>>1)+j)-0x20;
            }
            byte0 = twoNotes & 0xff;        // Frequency
            byte1 = twoNotes>>8 & 0xff;     // Delay & Duration
            twoNotes=twoNotes>>16;
            double frequency = freq[byte0 & 0x7f];
            if (byte0 >127) // If bit7 is set, there is a delay before note
                semiQuaver += (byte1>>4 & 0x0f) + 1;    // Note start time
            int duration = (byte1 & 0x0f) + 1;          // Note duration
            // Calculate start and stop times in terms of samples
            int start = (int)( (semiQuaver)*15f/tempo*RATE );
            int stop  = (int)( (semiQuaver+duration)*15f/tempo*RATE );

            // Calculate each sample and render it into the music loop
            for (int j=start; j<stop; j++) {
                double time = (j-start)/RATE;   // Time (seconds)                    
                // Triangle fundamental + sinusoidal 4th and 6th harmonics
                double sin4 = (Math.sin(Math.PI*frequency*time*8.0));
                double sin6 = (Math.sin(Math.PI*frequency*time*12.0));
                double triangle = 2.0*((frequency*time) % 1.0);
                if (triangle > 1.0) triangle = 2.0 - triangle;
                triangle = 2.0*triangle - 1.0;
                double sample = 0.5*triangle + 0.2*sin4 + 0.2*sin6;                    
                // Amplitude modulate the lot at about 2Hz
                sample *= 0.6 +0.4*(time % 0.5)/0.5;                   
                // Apply attack and decay to avoid clicks
                if (j-start<1000) sample *= (j-start)/1000f;
                if (stop-j<1000) sample *= (stop-j)/1000f;                   
                // Add the 16 bit sample to loop
                int data = (musicLoop[2*j]<<8)+(musicLoop[2*j+1] & 0xff);
                data += AMPLITUDE*sample;
                musicLoop[2*j+1] = (byte)(data & 0xff);
                musicLoop[2*j] = (byte)((data>>8) & 0xff);
            }
        }
        try {
            // Initialise Sound System
            AudioFormat audioFormat = new AudioFormat(RATE, 16, 1, true, true);
            DataLine.Info info = new DataLine.Info(Clip.class, audioFormat);
            music = (Clip)AudioSystem.getLine(info);
            music.open(audioFormat, musicLoop, 0, musicLoop.length);
            music.loop(Clip.LOOP_CONTINUOUSLY);
        } catch (Exception e) {
            e.printStackTrace(); // Display error, but keep going
        }
        
        // Screen
        BufferedImage screen = new BufferedImage(
            SCREENWIDTH, SCREENHEIGHT, BufferedImage.TYPE_INT_RGB);
        int[] screenData =
            ((DataBufferInt)screen.getRaster().getDataBuffer()).getData();
        Graphics g;
        setSize(SCREENWIDTH, SCREENHEIGHT);
        show();
        
            while (isVisible()) {
                int position = music.getFramePosition() % musicLoop.length;
                for (int x=0;x<SCREENWIDTH*20;x++) {
                    int xx = x/20;
                    int p = (2*position + 2*x) % musicLoop.length;
                    int note = 256*musicLoop[p] + (musicLoop[p+1] & 0xff);

                    for (int y=0; y<SCREENHEIGHT;y++) {
                        screenData[xx+y*SCREENWIDTH] = 0;
                    }
                    int yy = SCREENHEIGHT/2+note/32;
                    if (yy<0) yy = 0;
                    if (yy>=SCREENHEIGHT) yy = SCREENHEIGHT-1;
                    screenData[xx+yy*SCREENWIDTH] = 0xffffff;
                }
            g = getContentPane().getGraphics();
            g.drawImage(screen, 0, 0, null);
            g.dispose();
            }
    }
    
    /**
     * @param args the command line arguments
     */
    public static void main(String[] args) {
        // TODO code application logic here
        new S();
        System.exit(0);
    }
    
}

[quote]I don’t suppose you could add comments to the code to help those of us (namely me) who are total noobes to java sound api and sound generation in general?
[/quote]
I really should clean it up a bit more and add some comments. Basically it has 8 channels of an oscillator combined with noise. Each character of the string is an instruction to either set the frequency of the oscillator, the amplitude of the channel or a delay. Combining the commands and coupled with a bunch of static settings for each channel allows you to create sound effects or music.

[quote]It stuttered badly on my machine until I reduced the sample rate to a quarter to the value.
[/quote]
Yeah, it is reasonably processor intensive and many Java implementations have sound that simply will wont work real time. The main culprit of this is the Java Sound Mixer, I’m not sure what it is doing but you can’t get it to work without a massive buffer which equals seconds of delay.

In addition to lowering the sample rate I suggest you increase the buffer size of the line by changing this line:
dest.open(format, 2048);
If you just want it for music and don’t care about latency you can set this to huge amount for glitch free playback.

Also you can print out AudioSystem.getMixer(null).getMixerInfo().getName() and check it isn’t the Java Sound mixer.

Since Java 1.5 is the minimum target, I wonder whether it is worth re-considering the use of MIDI (without a soundbank). This drops through to using underlying hardware MIDI support when no soundbank is present and was hidiously broken in Java 1.4. However I wonder whether this works now. Any thoughts?

I think it fair to assume that on a reasonable number of systems MidiSystem.getSynthesizer() will return a valid synth. Going forward a rule could be added that the Java Midi is not forbidden but is use at your own risk and the judges aren’t required to get it working. However as the entire javax.sound.midi package is on notice as likely being removed from future JRE’s it seems that we would be be back to roll your own fairly soon anyway.

New version up. This one is a bit better at making sound effects and also cuts down the size of songs dramatically (1 byte per note rather than 2). I have also cleaned up the source a bit and included some comments.

I am not happy with the explosion effects but the others are sounding ok. There are 8 songs in this version (more to come). The synthesizer has a fair amount of static settings and could be set up differently for different types of effects/music. At the moment the 8 channels are programmed for a balance of FX/music:

1 - Lead melody
2 - Harmony
3 - Bass
4 - Snare
5 - Kick
6 - FX Shoot
7 - FX Boing
8 - FX Explode

This is pretty cool indeed!!.. The version you had a couple of weeks ago was very stuttery, but this seems to play stutter free!!

Actually 2 of the sounds would fit perfectly well into my next 4K shootem-up game… Is the source for general use? I wouldn’t want to use any of it without you getting the credit… I did investigate sound a few weeks ago but seemed to complicated/unstable to use… but slowly changing my mind, partly due to this post…

ps. The source in above link gives me 2 compiler errors (lines 168, 169… Casting cmd to Integer seems to fix).

[quote]Is the source for general use?
[/quote]
Go right ahead, hereby placed in the public domain. Probably should have made that clear in the initial post. I would be very pleased if an entry or two this year used it, I think sound/music is much more important than many people realize.

[quote]ps. The source in above link gives me 2 compiler errors (lines 168, 169… Casting cmd to Integer seems to fix).
[/quote]
Autoboxing issue I assume, what compiler are you using? I should probably add the casts just for backwards compatibility I don’t think it changes the class files.

Awesome :o
But the Source from new Version seems to have errors.
The Compiler is complaining about cmds.peek() and cmds.poll (around 168).

[quote]But the Source from new Version seems to have errors.
The Compiler is complaining about cmds.peek() and cmds.poll (around 168).
[/quote]
I have added explicit casts, should work on older compilers now.

Edit: Added intValue() calls, rather. I’m pretty sure this doesn’t change the class file size as I assume the autoboxing is just inserting the code for me.

[quote]Go right ahead, hereby placed in the public domain.
[/quote]
Thats great news… Thanks alot. Will try and use this in my next game. This may be a daft question, but I guess will need to play Sounds/Music in a new Thread? (i.e. I guess playing them directly from my main loop will cause stutter?)…

ps. New version still doesn’t compile… I think you need to change your cast from (int) to (Integer). Am using 1.5 btw. With the (Integer) cast this version seems to work ok in Eclipse for me. Thanks again :slight_smile:

[quote]This may be a daft question, but I guess will need to play Sounds/Music in a new Thread?
[/quote]
You can use it in the main thread and in fact it can take the place of timing code. The main loop runs at almost exactly 64 times per second (you can easily modify but the music tempo will change).
If you do too much processing the sound may stutter and it will need it’s own thread or it’s buffer size increased.

[quote]ps. New version still doesn’t compile…
[/quote]
Fixed now, got slightly confused between generics/autoboxing (need more coffee).

Just wanted to let you know, your sound app brought a smile to my face. Must be the nerd/classic-gamer in me or something.

Thanks. ;D

[quote]Just wanted to let you know, your sound app brought a smile to my face. Must be the nerd/classic-gamer in me or something.
[/quote]
I just picked a couple of my old favorites. NES tunes are quite easy to convert because it had a sound chip with limitations (3 monophonic oscillators + noise) that match up fairly well with my synth.

Very cool! I’ll have to try this out sometime soon. :slight_smile:

It is pretty damn cool indeed!! Last night I inserted the code into my new 4K Shootem up game (player fire, enemy explosion and 1 music track)… Although not perfect (needs some fine-tuning) it certainly did give the game an extra ‘cool’ factor. Am gonna do my extra best to try and keep this in (size permitting)…

I found a few issues (none of which I have had the time to investigate yet)…

  • After running Moogies Tool (Obfuscator) the sound didn’t work… (No idea why)!
  • The game was a more stuttery (but as you said, this may be just a case of finetuning some buffer sizing)…
  • It ran ok in Appletviewer but in Browser it was approx 10times a slow (not sure why)… Changing the Thread.sleep(16) to 8 made it quicker in the Browser.

Ok, have to admit, at first glance, have no idea how you get the UTF strings into music… Is there an explanation anywhere, Ideally would like to create my own track, if only I knew how…

I reckon I need another week to program in some gameplay and will post a demo…

Here is the quick and nasty converter I used to convert some NES midi files from http://www.vgmusic.com:

The converter takes two arguments, first the midi file and second a channel map. The channel map is the midi channel to scan for note commands for the synth channel.

Eg:
MidiConvert mario.mid 1,2,3,10

Would scan the file mario.mid and any messages on Midi channel 1 would go to the synth channel 1. Messages on midi channel 10 would go to synth channel 4. etc.

Note that the default synth config channels are set up for:
1 - Melody
2 - Harmony
3 - Bass
4 - Snare Drum
5 - Kick Drum

Also only a very small subset of midi is compatible with my synth. It is monophonic, not velocity sensitive, doesn’t support pitch bending, has no note-off capability. More complicated midi files will come out sounding rubbish.


import java.io.File;
import java.io.IOException;
import java.util.ArrayList;

import javax.sound.midi.InvalidMidiDataException;
import javax.sound.midi.MidiEvent;
import javax.sound.midi.MidiMessage;
import javax.sound.midi.MidiSystem;
import javax.sound.midi.Sequence;
import javax.sound.midi.ShortMessage;
import javax.sound.midi.Track;

public class MidiConvert {

	private static final int QUATER_NOTE_STEPS = 32;
	private static final int QUATER_NOTES_PER_SEC = 2;

	public static void main(String[] args) {
		if (args.length < 2){
			System.out.println("Usage: MidiConvert filename channelmap");
			return;
		}
		
		MidiConvert mc = new MidiConvert();
		try {
			Sequence sequence = MidiSystem.getSequence(new File(args[0]));
			printSequenceInfo(sequence);
			String[] channelFields = args[1].split("\\,");
			int[] channelMap = new int[channelFields.length];
			for(int i = 0; i < channelMap.length; i++){
				channelMap[i] = Integer.parseInt(channelFields[i]);
			}
			// 0 = melody
			// 1 = harmony
			// 2 = bass
			// 3 = beat
			String s = mc.process(sequence, channelMap);
			System.out.println(s);
		} catch (InvalidMidiDataException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	public static void printSequenceInfo(Sequence sequence){
		int events[] = new int[16];
		for(Track track : sequence.getTracks()){
			for(int i = 0; i < track.size(); i++){
				MidiMessage m = track.get(i).getMessage();
				if (m instanceof ShortMessage){
					events[((ShortMessage)m).getChannel()]++;
				}
			}
		}
		for(int i = 0; i < events.length; i++){
			System.out.println("Ch " + i + ": " + events[i] + " events");
		}
	}

	private ArrayList<Integer> commands;


	public int getStep(Sequence sequence, long midiTick) {
		float timing = sequence.getDivisionType();
		int res = sequence.getResolution();
		if (timing == Sequence.PPQ) {
			return (int) ((midiTick * QUATER_NOTE_STEPS) / res);
		} else {
			int stepsPerSec = QUATER_NOTE_STEPS * QUATER_NOTES_PER_SEC; // assume 120bpm
			int ticksPerStep = Math.round((timing * sequence.getResolution())/ stepsPerSec);
			return (int) (midiTick / ticksPerStep);
		}
	}

	public void noteOn(int channel, int key, int velocity) {
		if (velocity != 0) {
			key -= 36;
			while (key < 0) {
				System.out.println("Adjusting note into range +12");
				key += 12;
			}
			while (key > 63) {
				System.out.println("Adjusting note into range -12");
				key -= 12;
			}
			velocity = (velocity >> 2);
			commands.add((1 << 6) | key);
		}
	}

	public String process(Sequence sequence, int[] channelMap) {
		commands = new ArrayList<Integer>();
		for (int outChannel = 0; outChannel < channelMap.length; outChannel++) {
			if (channelMap[outChannel] < 0) {
				continue;
			}
			int lastStep = 0;
			commands.add(0xC080 | outChannel);
			for (Track track : sequence.getTracks()) {
				for (int i = 0; i < track.size(); i++) {
					MidiEvent event = track.get(i);
					if (event.getMessage() instanceof ShortMessage) {
						ShortMessage sm = (ShortMessage) event.getMessage();
						if (sm.getChannel() == channelMap[outChannel] && 
							sm.getCommand() == ShortMessage.NOTE_ON &&
							sm.getData2() != 0) {
							int step = getStep(sequence, event.getTick());
							int delta = step - lastStep;
							while (delta > 63) {
								commands.add(63);
								delta -= 63;
							}
							if (delta > 0) {
								commands.add(delta);
							}
							lastStep = step;
							noteOn(outChannel, sm.getData1(), sm.getData2());
						}
					}
				}
			}
		}
		StringBuilder sb = new StringBuilder();
		sb.append("String data = \"");
		for (int i = 0; i < commands.size(); i++) {
			int val = commands.get(i);
			if (val == 10){
				sb.append("\\n");
			} else if (val == 13){
				sb.append("\\r");
			} else if (val == 34){
				sb.append("\\\"");
			} else if (val == 92){
				sb.append("\\\\");
			} else if (val > 31 && val < 127){
				sb.append((char)val);
			} else {
				String ucode = Integer.toHexString(val);
				while (ucode.length() < 4) ucode = "0" + ucode;
				sb.append("\\u" + ucode);
			}
		}
		System.out.println();
		sb.append("\";");
		return sb.toString();
	}
}