Author: Yoon Hee Kim

Initial Idea

For my final project, I started off with a very vague idea that I wanted to combine colours and sounds and somehow recreate synesthesia. I initially thought of making it as small, physically, as having a melodica and a screen but after talking with Aaron, I increased the size of the project by using a piano instead.

 

Stress Inducing Wires

The process of attaching the wires inside the piano was a series of struggles.

After many nights of being stressed, having mental breakdowns and making stupid mistakes in the Art Centre, I was finally able to get every wire in place. I never thought sticking some wires could cause someone so much trouble. At least now I can confidently say that I can solder.

 

Code

Many thanks to Aaron and Daniel Shiffman for “helping me write” the code.

Arduino:

int noteC = 0;
int noteD = 0;
int noteE = 0;
int noteF = 0;
int noteG = 0;
int noteA = 0;
int noteB = 0;
int noteCC = 0;
int inByte = 0;

void setup() {
  Serial.begin(9600);
  pinMode(2, INPUT);
  pinMode(3, INPUT);
  pinMode(4, INPUT);
  pinMode(5, INPUT);
  pinMode(6, INPUT);
  pinMode(7, INPUT);
  pinMode(8, INPUT);
  pinMode(9, INPUT);
  establishContact();
}

void loop() {
  if (Serial.available() > 0) {
    inByte = Serial.read();
    noteC = digitalRead(2);
    noteD = digitalRead(3);
    noteE = digitalRead(4);
    noteF = digitalRead(5);
    noteG = digitalRead(6);
    noteA = digitalRead(7);
    noteB = digitalRead(8);
    noteCC = digitalRead(9);

    Serial.print(noteC);
    Serial.print(",");
    Serial.print(noteD);
    Serial.print(",");
    Serial.print(noteE);
    Serial.print(",");
    Serial.print(noteF);
    Serial.print(",");
    Serial.print(noteG);
    Serial.print(",");
    Serial.print(noteA);
    Serial.print(",");
    Serial.print(noteB);
    Serial.print(",");
    Serial.println(noteCC);
  }
}

void establishContact() {
  while (Serial.available() <= 0) {
    Serial.println("0,0,0,0,0,0,0,0");
  }
}

Processing:

import processing.serial.*; 
import codeanticode.syphon.*;

SyphonServer server;
Serial myPort;  

import toxi.geom.*;
import java.util.*;

PGL pgl;

Emitter emitter;
Vec3D gravity;
float floorLevel;

PImage particleImg;
PImage emitterImg;

int counter;

boolean noteState[]={false, false, false, false, false, false, false, false};
int noteCounter[] = {0, 0, 0, 0, 0, 0, 0, 0};
int notes[]={0, 0, 0, 0, 0, 0, 0, 0};
int countTarget = 10;

boolean ALLOWGRAVITY;    // add gravity vector?
boolean ALLOWPERLIN=true;     // add perlin noise flow field vector?
boolean ALLOWTRAILS;     // render particle trails?
boolean ALLOWFLOOR;      // add a floor?

color c;

void setup() {
  size( 600, 600, P3D );
  server = new SyphonServer(this, "Processing Syphon");
  smooth(4);
  colorMode( RGB, 1.0 );

  pgl         = ((PGraphicsOpenGL) g).pgl;  

  // Loads in a particle image from the data folder. Image size should be a power of 2.
  particleImg = loadImage( "emitter.png" );
  emitterImg  = loadImage( "emitter.png" );

  emitter     = new Emitter();
  gravity     = new Vec3D( 0, .35, 0 );    // gravity vector
  floorLevel  = 400;

  printArray(Serial.list());
  myPort = new Serial(this, Serial.list()[1], 9600);
  myPort.clear();
  myPort.bufferUntil('\n');
}

void draw() {
  background( 0 );
  perspective( PI/3.0, (float)width/(float)height, 1, 5000 );

  pgl.depthMask(false);

  pgl.enable( PGL.BLEND );
  pgl.blendFunc(PGL.SRC_ALPHA, PGL.ONE);

  emitter.exist();
  }
  
  float total=0;
  for (int noteNum = 0; noteNum < notes.length; noteNum++) {
    total+= notes[noteNum];
  }
  total/=notes.length;
  for (int noteNum = 0; noteNum < notes.length; noteNum++) {

    if (notes[noteNum]==1 ) {
      noteCounter[noteNum]++;
    } else if (notes[noteNum]==0) {
      noteCounter[noteNum]=0;
    }
    if (noteCounter[noteNum]>countTarget)
      noteState[noteNum]=true;
    else if (noteCounter[noteNum]==0)
      noteState[noteNum]=false;
  }
  
  int numParticles = 2;
  
  if (noteState[0] == true) {
    emitter.pos.x= 290;
    emitter.pos.y= 600;
    c = #B22222;
    emitter.addParticles( numParticles );
  }

  if (noteState[1] == true) {
    emitter.pos.x= 302;
    emitter.pos.y= 600;
    c = #FF3910;
    emitter.addParticles( numParticles );
  }

  if (noteState[2] == true) {
    emitter.pos.x= 314;
    emitter.pos.y= 600;
    c = #FF6347;
    emitter.addParticles( numParticles );
  }

  if (noteState[3] == true) {
    emitter.pos.x= 322;
    emitter.pos.y= 600;
    c = #90EE90;
    emitter.addParticles( numParticles );
  }
  
  if (noteState[4] == true) {
    emitter.pos.x= 332; 
    emitter.pos.y= 600;
    c = #87CEFA;
    emitter.addParticles( numParticles );
  }
  
  if (noteState[5] == true) {
    emitter.pos.x= 342;
    emitter.pos.y= 600;
    c = #2A51FF;
    emitter.addParticles( numParticles );
  }
  
  if (noteState[6] == true) {
    emitter.pos.x= 353;
    emitter.pos.y= 600;
    c = #8A2BE2;
    emitter.addParticles( numParticles );
  }
  
  if (noteState[7] == true) {
    emitter.pos.x= 360;
    emitter.pos.y= 600;
    c = #FF4B60;
    emitter.addParticles( numParticles );
  }

  counter ++;
  server.sendScreen();
}

void serialEvent(Serial myPort) {
  String myString = myPort.readStringUntil('\n');
  myString = trim(myString);
  int _notes[] = int(split(myString, ','));
  if (_notes.length==8) {
    notes=_notes;
  }
  myPort.write("A");
}

void renderImage(PImage img, Vec3D _pos, float _diam, color _col, float _alpha ) {
  pushMatrix();
  translate( _pos.x, _pos.y, _pos.z );
  tint(red(_col), green(_col), blue(_col), _alpha);
  imageMode(CENTER);
  image(img,0,0,_diam,_diam);
  popMatrix();
}

class Emitter{
  Vec3D pos;
  Vec3D vel;
  Vec3D velToMouse;
  
  color myColor;
  
  ArrayList particles;
  
  Emitter(  ){
    pos        = new Vec3D();
    vel        = new Vec3D();
    velToMouse = new Vec3D();
    
    myColor    = color( 1, 1, 1 );
    
    particles  = new ArrayList();
  }
  
  void exist(){
    setVelToMouse();
    findVelocity();
    setPosition();
    iterateListExist();
    render();
    
    pgl.disable( PGL.TEXTURE_2D );
    
    if( ALLOWTRAILS )
      iterateListRenderTrails();
  }
   
  void findVelocity(){
    vel.interpolateToSelf( velToMouse, .35 );
  }
  
  void setPosition(){
    pos.addSelf( vel );
    
    if( ALLOWFLOOR ){
      if( pos.y > floorLevel ){
        pos.y = floorLevel;
        vel.y = 0;
      }
    }
  }
  
  void iterateListExist(){
    for( Iterator it = particles.iterator(); it.hasNext(); ){
      Particle p = (Particle) it.next();
      if( !p.ISDEAD ){
        p.exist();
      } else {
        it.remove();
      }
    }
  }
  
  void iterateListRenderTrails(){
    for( Iterator it = particles.iterator(); it.hasNext(); ){
      Particle p = (Particle) it.next();
      p.renderTrails();
    }
  }

  void addParticles( int _amt ){
    for( int i=0; i<_amt; i++ ){
      particles.add( new Particle( pos, vel, c ) );
    }
  }
}

class Particle {
  int len;            // number of elements in position array
  Vec3D[] pos;        // array of position vectors
  Vec3D startpos;     // just used to make sure every pos[] is initialized to the same position
  Vec3D vel;          // velocity vector
  Vec3D perlin;       // perlin noise vector
  float radius;       // particle's size
  float age;          // current age of particle
  int lifeSpan;       // max allowed age of particle
  float agePer;       // range from 1.0 (birth) to 0.0 (death)
  float bounceAge;    // amount to age particle when it bounces off floor
  boolean ISDEAD;     // if age == lifeSpan, make particle die
  boolean ISBOUNCING; // if particle hits the floor...
  color col;

  Particle( Vec3D _pos, Vec3D _vel, color _c ) {
    radius      = random( 10, 50 );
    len         = (int)( radius );
    pos         = new Vec3D[ len ];
    col = _c;
    startpos    = new Vec3D( _pos.add( new Vec3D().randomVector().scaleSelf( random( 5.0 ) ) ) ); 

    for ( int i=0; i<len; i++ ) {
      pos[i]    = new Vec3D( startpos );
    }

    vel   = new Vec3D( _vel.scale( .5 ).addSelf( new Vec3D().randomVector().scaleSelf( random( 1 ) ) ) );
    vel.x = random(-.1, .1);
    vel.y = random(-5, -1);

    perlin      = new Vec3D();

    age         = 0;
    bounceAge   = 2;
    lifeSpan    = (int)( radius )*10;
  }

  void exist() {
    if ( ALLOWPERLIN )
      findPerlin();

    findVelocity();
    setPosition();
    render();
    setAge();
  }

  void findPerlin() {
    float xyRads      = getRads( pos[0].x, pos[0].z, 10.0, 30.0 );
    float yRads       = getRads( pos[0].x, pos[0].y, 10.0, 30.0 );
    perlin.set( cos(xyRads), -sin(yRads), sin(xyRads) );
    perlin.scaleSelf( .5 );
  }

  void findVelocity() {
    if ( ALLOWGRAVITY )
      vel.addSelf( gravity );

    if ( ALLOWPERLIN ) {
      if (agePer<.75 && agePer > 0 ) {
        Vec3D spread = new Vec3D(random(-1, 1), 0, random(-.5, .5)); 
        vel.addSelf(spread);
      }
    }
  }

  void setPosition() {
    for ( int i=len-1; i>0; i-- ) {
      pos[i].set( pos[i-1] );
    }
    pos[0].addSelf( vel );
  }

  void render() {
    renderImage(particleImg, pos[0], radius * agePer, col, 1.0 );
  }

  void renderTrails() {
    float xp, yp, zp;
    float xOff, yOff, zOff;
    beginShape(QUAD_STRIP);
    for ( int i=0; i<len - 1; i++ ) {
      float per     = 1.0 - (float)i/(float)(len-1);
      xp            = pos[i].x;
      yp            = pos[i].y;
      zp            = pos[i].z;

      if ( i < len - 2 ) {
        Vec3D perp0 = pos[i].sub( pos[i+1] );
        Vec3D perp1 = perp0.cross( new Vec3D( 0, 1, 0 ) ).normalize();
        Vec3D perp2 = perp0.cross( perp1 ).normalize();
        perp1 = perp0.cross( perp2 ).normalize();

        xOff        = perp1.x * radius * agePer * per * .1;
        yOff        = perp1.y * radius * agePer * per * .1;
        zOff        = perp1.z * radius * agePer * per * .1;

        fill( per, per*.25, 1.0 - per, per * .5);
        noStroke();
        vertex( xp - xOff, yp - yOff, zp - zOff );
        vertex( xp + xOff, yp + yOff, zp + zOff );
      }
    }
    endShape();
  }

  void setAge() {
    if ( ALLOWFLOOR ) {
      if ( ISBOUNCING ) {
        age += bounceAge;
        bounceAge ++;
      } else {
        age += .25;
      }
    } else {
      age ++;
    }

    if ( age > lifeSpan ) {
      ISDEAD = true;
    } else {
      // When spawned, the agePer is 1.0.
      // When death occurs, the agePer is 0.0.
      agePer = 1.0 - age/(float)lifeSpan;
    }
  }
}

 

Visual

I continuously changed around the png being emitted, the location of emission and the colour of the emitters. In the end, I settled with a png with one small bubble rather than the bulky bubble groups that I initially used. The location of the emission was in line with the keys.

 

Showcase

The showcase, personally, was quite a stressful event as I was constantly worried about my wires when watching people bang on the keys. It was equally stressful when the bubbles didn’t come up because people were playing too fast – maybe I should have had Adagio projected on the screen for subtle warning.

I got a lot of interesting feedback but the two comments that were the most interesting were:

1. it would be useful to apply this to music education / piano education for children.

2. “I would actually get something like this for my future children. Like seriously.”

But overall, it was very pleasing and satisfying to watch people ooh and ahh as they play the piano.

Improvements

If I were to improve this project, I could extend the effect to all keys on the piano and find a way to make switches work even when the keys are pressed fast.

The responses from the users were mainly

Ooh that’s cool!

and

Oh wait, what did I do?

due to nice bubble columns that came up on the screen when the keys are pressed and the nice bubble that didn’t stop coming when the keys weren’t pressed.

The first user testing was done before things were working quite the way I wanted it to so I made some changes before the second user testing regardless of the feedback from the first user testing, The changes I made before the second user testing were:

1. To make the display for the notes distinct from one another,

• Spaced out the x position of each bubble columns.
• Assigned each notes to a colour.

2. To make the display more clearly visible

• Put a white piece of cloth over the piano.

After the second user testing, I took the cloth down and just put it over the piano to see which set up looked better, then settled with not hanging the cloth on the wall as the user testing showed that the user(s) liked that better. Before the second user testing, I wanted to make the colours of each columns to be different and stay different but had failed to do so. When I brought it up with the user after the user testing, he said he likes how the colours of all the columns change to the colour of the new pressed key but I am unsure as to how much the user feedback should count towards the project.

The reading was relatively easy to understand and informative as well as interesting as it gives specific examples such as limbo after explaining the different computer vision techniques.

The part about detecting smile was especially interesting. Before learning about how computers work, I had thought of the smile mode on phone cameras that take pictures when a person smiles as detecting happiness in a person. After doing this reading, it makes much more sense to understand it as detection of muscle movement in certain direction.

I got the idea for my final project from synesthesia which is when hearing sounds triggers seeing colours. Although not exactly the same as synesthesia, I borrowed the concept of experiencing colours through sound for this project. My plan is to have a musical instrument which allows the users to paint pictures through playing the instrument. Every user and every attempt at playing will produce different paintings

 

Materials needed:

• Laptop
• Musical instrument (build a new one or use an already existing one)
• Screen
• Arduino

 

• The users will be asked to play an instrument which is fairly easy to work out how to play (such as a piano keyboard). 
• Each key/note will be assigned a colour and the shade of the colour will change according to how long the key is pressed down.
• Blotch of colours in varying size and location will appear on the screen as the users play the instrument.

I originally took this course hoping to gain some understanding of how computers and machines work (and I think I have – ish). But at the same time, ironically, instead of simply being amazed at all the cool stuff computers and programming allows us to do, I have come to appreciate the human qualities more. This might sound very random to many people and I don’t know how accurate it would be to describe programming as leaving the time-consuming processes to computers which make less mistakes than humans do. But my point is, getting a glimpse of how computers work allowed me to highlight the differences between humans and machines and came to appreciate the human capabilities I took for granted as well as technology.

When I tried to write a very simple code of less than 20 lines with the limited knowledge that I have, an error that I could not work out kept on occurring. I took multiple approaches to fix the code. First, I went over line by line and see what wasn’t working. Then, I opened a new screen and started writing from scratch. Finally, I got someone else to write the code for me and tried to replicate it. After failing to write 20 lines of code correctly, I realised that I had spelt ‘colour’ instead of ‘color’. The fact that computers which can carry out such complex processes without making a mistake, could not recognise ‘colour’ as ‘color’ made computers seem ridiculous to me but at the same time thought that humans are actually capable of a lot of things that don’t happen deservedly for machines.

I further developed the tickle project that I did for last week by making ‘Yoon Hee’ run away from the hands that are trying to tickle it. The further from the centre of the word, the faster it moves away.

I used two potentiometers as the controllers to replace the cursor and change the x, y positions of the hands.

Here is the code for Arduino:

void setup() {
// put your setup code here, to run once:
pinMode(button, INPUT);
Serial.begin(9600);
Serial.println("0,0");
}

void loop() {
if (Serial.available() > 0) {
int inByte = Serial.read();

int sensor = analogRead(A0);
delay(0);
int sensor2 = analogRead(A1);
delay(0);
Serial.print(sensor);
Serial.print(',');
Serial.println(sensor2);
}
}

 

Here is the code for Processing:

import processing.serial.*;

Serial myPort;
PFont f;
String word = "Yoon Hee";
PImage img;

float x = 190;
float y = 180;
float speedX, speedY, xAccel, yAccel;

void setup() {
size(640, 360);
printArray(Serial.list());
String portName = Serial.list()[3];
println(portName);
myPort = new Serial (this, portName, 9600);
myPort.clear();
myPort.bufferUntil('\n');
background(255);
f = createFont("Monaco", 50);
textFont(f, 50);
noStroke();
speedX = speedY = xAccel = yAccel= 0;
img = loadImage("hand.png");
}

void draw() {
fill(204);
rect(0, 0, width, height);
fill(0);

image(img, mouseX-25, mouseY-10, 50, 50);

float textwidth = textWidth(word);
float centerX, centerY;
centerX = x + textwidth/2;
centerY = y - 35/2;
float dirX = centerX - mouseX;
float dirY = centerY - mouseY;

boolean inside =false;
if ((mouseX >= x && mouseX <= x+textwidth) &&
(mouseY >= y-35 && mouseY <= y)) {
inside=true;
}

PVector distance = new PVector(dirX, dirY);
float d = distance.mag();

println(distance +" norm");
if (d<textwidth*.75) {
distance.normalize();
xAccel = dirX*.005*(1/distance.mag());
yAccel = dirY*.005*(1/distance.mag());
}

boolean tickle=false;
if (mousePressed == true && inside == true) {
tickle=true;
xAccel=speedX=0;
yAccel=speedY=0;
}

for (int i=0; i<word.length(); i++) {
char c = word.charAt(i);

float letterX=x+textWidth(c)*i;
float tickleX = 0;
float tickleY = 0;
if (tickle) {
//tickleX=noise();
tickleY= noise((.5*frameCount+i)*.5)*30-15; //sin(frameCount*0.2+(i+1))*5;
}
text(c, letterX+tickleX, y+tickleY);
}

//text(word, x, y);
x += speedX;//random(-2, 2);
y += speedY;//random(-2, 2);
speedX *= .8;
speedY *= .8;

speedX += xAccel;
speedY += yAccel;
xAccel = yAccel= 0;

//bounce off the edges
if (y<35)
speedY*=-1;
if (y>height)
speedY*=-1;
if (x<0)
speedX*=-1;
if (x>width-textwidth)
speedX*=-1;
}

void serialEvent(Serial myPort) {
String x = myPort.readStringUntil('\n');
x = trim(x);
if (x!= null) {
int values[] = int(split(x, ','));
mouseX = (int)map(values[0], 0, 1023, 0, width);
mouseY = (int)map(values[1], 0, 1023, 0, height);
}

myPort.write('0');
}

 

I found the part related to the social data especially interesting as I had come across very similar content in a book about psychology and statistics. The idea that tweets can be used to predict movie box-office revenue was very interesting as seemingly independent data can be used to make predictions which can result in huge economic benefit. To mention that so much of the work that humans used to do has been replaced by machines and technology is probably stating the obvious, however, this reading made me really realise that so much can be done with the collected and stored data and wonder how much more can be done with the new technology to come.

Following with my assignment from last week which made several shapes vibrate, I wanted to make the letters vibrate for this week’s assignment.

I also wanted to incorporate some human quality into the project so I thought of the  vibrating motion as “tickling” and I made the letters “run away” from the mouse which “tickles” them.

import geomerative.*;
String text = "Yoon Hee";

float x = 190;
float y = 180;

void setup() {
size(640, 360);
background(255);
textSize(50);
noStroke();
}

void draw() {
fill(204);
rect(0, 0, width, height);
fill(0);
text("Yoon Hee", x, y);

if (mousePressed == true) {
x += random(-2, 2); 
y += random(-2, 2); 
text("Yoon Hee", x, y);
} else {
text("Yoon Hee", x, y);
}
}

void mousePressed() {
if ((mouseX >= x) && (mouseX <= x+30) &&
(mouseY >= y-5) && (mouseY <= y)) {
x += random(-2, 2); 
y += random(-2, 2);
}
}

 

For this assignment, I have worked with class and objects.

I based the idea on the work I replicated for Assignment 6 and came up with the code with the help of Getting started with Processing and online resources.

Here is the code that I used:

CrazyShape circle1, circle2, square3;

void setup() {
size(500,500);
smooth();
circle1 = new CrazyShape(20, 1.5, color(random(255), 125, 0));
circle2 = new CrazyShape(10, 3, color(255, 0, random(155)));
square3 = new CrazyShape(30, 2.5, color(0, random(125), 255));
}

void draw() {
circle1.move();
circle1.displayCircle();
circle2.move();
circle2.displayCircle();
square3.move();
square3.displaySquare();
}

CrazyShape

class CrazyShape {
float x;
float y;
int diameter;
float speed;
color c;

CrazyShape(int id, float ispeed, color ic) {
x = random(width);
y = random(height);
speed = ispeed;
diameter = id;
c = ic;
}

void move() {
x += random(-speed, speed);
y += random(-speed, speed);

if (x < 0+diameter) {
x = random(width);
c = color(random(255), random(255), random(255));
} else if (x > width-diameter) {
x = random(width);
c = color(random(255), random(255), random(255));
}

if (y < 0+diameter) {
y = random(height);
c = color(random(255), random(255), random(255));
} else if (y > height-diameter) {
y = random(height);
c = color(random(255), random(255), random(255));
}
}

void displayCircle() {
fill(c);
ellipse(x, y, diameter, diameter);
}

void displaySquare() {
fill(c);
rect(x, y, diameter, diameter);
}
}

       

I replicated the work done by Edward Zajec.

Here is the code that I used:

void setup() {
size(380, 500);
}

void draw() {
background(255);
noFill();
smooth();
strokeWeight(1);
stroke(0);

//circle1
for (int i=25; i<359; i+= 25) {
ellipse(140, 150, i, i);
}

//circle2
for (int i=10; i<200; i+= 8) {
ellipse(355, 380, i, i);
}

//square1
stroke(150);
pushMatrix();
{
rotate(PI/5);
for (int i=2; i<120; i+=4) {
rectMode(CENTER);
rect(250, -20, i, i);
}
popMatrix();
}

//circle3
stroke(100);
for (int i=2; i<80; i+= 3) {
ellipse(280, 180, i, i);
}

//square2
stroke(120);
pushMatrix();
{
rotate(PI/6);
for (int i=2; i<200; i+=6) {
rectMode(CENTER);
rect(380, 180, i, i);
}
popMatrix();
}

//triangle
stroke(0);
for (int i=2; i<30; i++){
triangle(40,15,44+i,17+i, 37-i, 13+i/2);
triangle(45,15,48+i,13-i,48+i,17+i);
triangle(30,13,7+i/2,25-i/4,68-i,-27+i);
}
}