Initial Idea
For my final project, I started off with a very vague idea that I wanted to combine colours and sounds and somehow recreate synesthesia. I initially thought of making it as small, physically, as having a melodica and a screen but after talking with Aaron, I increased the size of the project by using a piano instead.
Stress Inducing Wires
The process of attaching the wires inside the piano was a series of struggles.
After many nights of being stressed, having mental breakdowns and making stupid mistakes in the Art Centre, I was finally able to get every wire in place. I never thought sticking some wires could cause someone so much trouble. At least now I can confidently say that I can solder.
Code
Many thanks to Aaron and Daniel Shiffman for “helping me write” the code.
Arduino:
int noteC = 0;
int noteD = 0;
int noteE = 0;
int noteF = 0;
int noteG = 0;
int noteA = 0;
int noteB = 0;
int noteCC = 0;
int inByte = 0;
void setup() {
Serial.begin(9600);
pinMode(2, INPUT);
pinMode(3, INPUT);
pinMode(4, INPUT);
pinMode(5, INPUT);
pinMode(6, INPUT);
pinMode(7, INPUT);
pinMode(8, INPUT);
pinMode(9, INPUT);
establishContact();
}
void loop() {
if (Serial.available() > 0) {
inByte = Serial.read();
noteC = digitalRead(2);
noteD = digitalRead(3);
noteE = digitalRead(4);
noteF = digitalRead(5);
noteG = digitalRead(6);
noteA = digitalRead(7);
noteB = digitalRead(8);
noteCC = digitalRead(9);
Serial.print(noteC);
Serial.print(",");
Serial.print(noteD);
Serial.print(",");
Serial.print(noteE);
Serial.print(",");
Serial.print(noteF);
Serial.print(",");
Serial.print(noteG);
Serial.print(",");
Serial.print(noteA);
Serial.print(",");
Serial.print(noteB);
Serial.print(",");
Serial.println(noteCC);
}
}
void establishContact() {
while (Serial.available() <= 0) {
Serial.println("0,0,0,0,0,0,0,0");
}
}
Processing:
import processing.serial.*;
import codeanticode.syphon.*;
SyphonServer server;
Serial myPort;
import toxi.geom.*;
import java.util.*;
PGL pgl;
Emitter emitter;
Vec3D gravity;
float floorLevel;
PImage particleImg;
PImage emitterImg;
int counter;
boolean noteState[]={false, false, false, false, false, false, false, false};
int noteCounter[] = {0, 0, 0, 0, 0, 0, 0, 0};
int notes[]={0, 0, 0, 0, 0, 0, 0, 0};
int countTarget = 10;
boolean ALLOWGRAVITY; // add gravity vector?
boolean ALLOWPERLIN=true; // add perlin noise flow field vector?
boolean ALLOWTRAILS; // render particle trails?
boolean ALLOWFLOOR; // add a floor?
color c;
void setup() {
size( 600, 600, P3D );
server = new SyphonServer(this, "Processing Syphon");
smooth(4);
colorMode( RGB, 1.0 );
pgl = ((PGraphicsOpenGL) g).pgl;
// Loads in a particle image from the data folder. Image size should be a power of 2.
particleImg = loadImage( "emitter.png" );
emitterImg = loadImage( "emitter.png" );
emitter = new Emitter();
gravity = new Vec3D( 0, .35, 0 ); // gravity vector
floorLevel = 400;
printArray(Serial.list());
myPort = new Serial(this, Serial.list()[1], 9600);
myPort.clear();
myPort.bufferUntil('\n');
}
void draw() {
background( 0 );
perspective( PI/3.0, (float)width/(float)height, 1, 5000 );
pgl.depthMask(false);
pgl.enable( PGL.BLEND );
pgl.blendFunc(PGL.SRC_ALPHA, PGL.ONE);
emitter.exist();
}
float total=0;
for (int noteNum = 0; noteNum < notes.length; noteNum++) {
total+= notes[noteNum];
}
total/=notes.length;
for (int noteNum = 0; noteNum < notes.length; noteNum++) {
if (notes[noteNum]==1 ) {
noteCounter[noteNum]++;
} else if (notes[noteNum]==0) {
noteCounter[noteNum]=0;
}
if (noteCounter[noteNum]>countTarget)
noteState[noteNum]=true;
else if (noteCounter[noteNum]==0)
noteState[noteNum]=false;
}
int numParticles = 2;
if (noteState[0] == true) {
emitter.pos.x= 290;
emitter.pos.y= 600;
c = #B22222;
emitter.addParticles( numParticles );
}
if (noteState[1] == true) {
emitter.pos.x= 302;
emitter.pos.y= 600;
c = #FF3910;
emitter.addParticles( numParticles );
}
if (noteState[2] == true) {
emitter.pos.x= 314;
emitter.pos.y= 600;
c = #FF6347;
emitter.addParticles( numParticles );
}
if (noteState[3] == true) {
emitter.pos.x= 322;
emitter.pos.y= 600;
c = #90EE90;
emitter.addParticles( numParticles );
}
if (noteState[4] == true) {
emitter.pos.x= 332;
emitter.pos.y= 600;
c = #87CEFA;
emitter.addParticles( numParticles );
}
if (noteState[5] == true) {
emitter.pos.x= 342;
emitter.pos.y= 600;
c = #2A51FF;
emitter.addParticles( numParticles );
}
if (noteState[6] == true) {
emitter.pos.x= 353;
emitter.pos.y= 600;
c = #8A2BE2;
emitter.addParticles( numParticles );
}
if (noteState[7] == true) {
emitter.pos.x= 360;
emitter.pos.y= 600;
c = #FF4B60;
emitter.addParticles( numParticles );
}
counter ++;
server.sendScreen();
}
void serialEvent(Serial myPort) {
String myString = myPort.readStringUntil('\n');
myString = trim(myString);
int _notes[] = int(split(myString, ','));
if (_notes.length==8) {
notes=_notes;
}
myPort.write("A");
}
void renderImage(PImage img, Vec3D _pos, float _diam, color _col, float _alpha ) {
pushMatrix();
translate( _pos.x, _pos.y, _pos.z );
tint(red(_col), green(_col), blue(_col), _alpha);
imageMode(CENTER);
image(img,0,0,_diam,_diam);
popMatrix();
}
class Emitter{
Vec3D pos;
Vec3D vel;
Vec3D velToMouse;
color myColor;
ArrayList particles;
Emitter( ){
pos = new Vec3D();
vel = new Vec3D();
velToMouse = new Vec3D();
myColor = color( 1, 1, 1 );
particles = new ArrayList();
}
void exist(){
setVelToMouse();
findVelocity();
setPosition();
iterateListExist();
render();
pgl.disable( PGL.TEXTURE_2D );
if( ALLOWTRAILS )
iterateListRenderTrails();
}
void findVelocity(){
vel.interpolateToSelf( velToMouse, .35 );
}
void setPosition(){
pos.addSelf( vel );
if( ALLOWFLOOR ){
if( pos.y > floorLevel ){
pos.y = floorLevel;
vel.y = 0;
}
}
}
void iterateListExist(){
for( Iterator it = particles.iterator(); it.hasNext(); ){
Particle p = (Particle) it.next();
if( !p.ISDEAD ){
p.exist();
} else {
it.remove();
}
}
}
void iterateListRenderTrails(){
for( Iterator it = particles.iterator(); it.hasNext(); ){
Particle p = (Particle) it.next();
p.renderTrails();
}
}
void addParticles( int _amt ){
for( int i=0; i<_amt; i++ ){
particles.add( new Particle( pos, vel, c ) );
}
}
}
class Particle {
int len; // number of elements in position array
Vec3D[] pos; // array of position vectors
Vec3D startpos; // just used to make sure every pos[] is initialized to the same position
Vec3D vel; // velocity vector
Vec3D perlin; // perlin noise vector
float radius; // particle's size
float age; // current age of particle
int lifeSpan; // max allowed age of particle
float agePer; // range from 1.0 (birth) to 0.0 (death)
float bounceAge; // amount to age particle when it bounces off floor
boolean ISDEAD; // if age == lifeSpan, make particle die
boolean ISBOUNCING; // if particle hits the floor...
color col;
Particle( Vec3D _pos, Vec3D _vel, color _c ) {
radius = random( 10, 50 );
len = (int)( radius );
pos = new Vec3D[ len ];
col = _c;
startpos = new Vec3D( _pos.add( new Vec3D().randomVector().scaleSelf( random( 5.0 ) ) ) );
for ( int i=0; i<len; i++ ) {
pos[i] = new Vec3D( startpos );
}
vel = new Vec3D( _vel.scale( .5 ).addSelf( new Vec3D().randomVector().scaleSelf( random( 1 ) ) ) );
vel.x = random(-.1, .1);
vel.y = random(-5, -1);
perlin = new Vec3D();
age = 0;
bounceAge = 2;
lifeSpan = (int)( radius )*10;
}
void exist() {
if ( ALLOWPERLIN )
findPerlin();
findVelocity();
setPosition();
render();
setAge();
}
void findPerlin() {
float xyRads = getRads( pos[0].x, pos[0].z, 10.0, 30.0 );
float yRads = getRads( pos[0].x, pos[0].y, 10.0, 30.0 );
perlin.set( cos(xyRads), -sin(yRads), sin(xyRads) );
perlin.scaleSelf( .5 );
}
void findVelocity() {
if ( ALLOWGRAVITY )
vel.addSelf( gravity );
if ( ALLOWPERLIN ) {
if (agePer<.75 && agePer > 0 ) {
Vec3D spread = new Vec3D(random(-1, 1), 0, random(-.5, .5));
vel.addSelf(spread);
}
}
}
void setPosition() {
for ( int i=len-1; i>0; i-- ) {
pos[i].set( pos[i-1] );
}
pos[0].addSelf( vel );
}
void render() {
renderImage(particleImg, pos[0], radius * agePer, col, 1.0 );
}
void renderTrails() {
float xp, yp, zp;
float xOff, yOff, zOff;
beginShape(QUAD_STRIP);
for ( int i=0; i<len - 1; i++ ) {
float per = 1.0 - (float)i/(float)(len-1);
xp = pos[i].x;
yp = pos[i].y;
zp = pos[i].z;
if ( i < len - 2 ) {
Vec3D perp0 = pos[i].sub( pos[i+1] );
Vec3D perp1 = perp0.cross( new Vec3D( 0, 1, 0 ) ).normalize();
Vec3D perp2 = perp0.cross( perp1 ).normalize();
perp1 = perp0.cross( perp2 ).normalize();
xOff = perp1.x * radius * agePer * per * .1;
yOff = perp1.y * radius * agePer * per * .1;
zOff = perp1.z * radius * agePer * per * .1;
fill( per, per*.25, 1.0 - per, per * .5);
noStroke();
vertex( xp - xOff, yp - yOff, zp - zOff );
vertex( xp + xOff, yp + yOff, zp + zOff );
}
}
endShape();
}
void setAge() {
if ( ALLOWFLOOR ) {
if ( ISBOUNCING ) {
age += bounceAge;
bounceAge ++;
} else {
age += .25;
}
} else {
age ++;
}
if ( age > lifeSpan ) {
ISDEAD = true;
} else {
// When spawned, the agePer is 1.0.
// When death occurs, the agePer is 0.0.
agePer = 1.0 - age/(float)lifeSpan;
}
}
}
Visual
I continuously changed around the png being emitted, the location of emission and the colour of the emitters. In the end, I settled with a png with one small bubble rather than the bulky bubble groups that I initially used. The location of the emission was in line with the keys.
Showcase
The showcase, personally, was quite a stressful event as I was constantly worried about my wires when watching people bang on the keys. It was equally stressful when the bubbles didn’t come up because people were playing too fast – maybe I should have had Adagio projected on the screen for subtle warning.
I got a lot of interesting feedback but the two comments that were the most interesting were:
1. it would be useful to apply this to music education / piano education for children.
2. “I would actually get something like this for my future children. Like seriously.”
But overall, it was very pleasing and satisfying to watch people ooh and ahh as they play the piano.
Improvements
If I were to improve this project, I could extend the effect to all keys on the piano and find a way to make switches work even when the keys are pressed fast.