Processing Sketch: Sound eating away at pixels.

Jackson Pollock

Piet Mondriaan

import ddf.minim.*;
import ddf.minim.analysis.*;
import ddf.minim.effects.*;
import ddf.minim.signals.*;
import ddf.minim.spi.*;
import ddf.minim.ugens.*;

PImage image;
float[] frequenciesR;
float[] frequenciesG;
float[] frequenciesB;
Minim minim = new Minim( this );
AudioOutput out = minim.getLineOut( Minim.STEREO);
Oscil waveR;
Oscil waveG;
Oscil waveB;
float mils;
int j=0 ;
float speed = 1;

void setup() {
  
  size(512,512);
  image = loadImage("PM.jpg");
  image.resize(512,512);
  int dimension = image.width * image.height;
  image.loadPixels();
  frequenciesR = new float[dimension];
  frequenciesG = new float[dimension];
  frequenciesB = new float[dimension];
  for(int i=0 ; i<dimension ; i++)
  { 
    frequenciesR[i] = red(image.pixels[i]);
    frequenciesG[i] = green(image.pixels[i]);
    frequenciesB[i] = blue(image.pixels[i]);
  }
  
  waveR = new Oscil( 0, 0.5f, Waves.SAW );
  waveR.patch( out );
  waveG = new Oscil( 0, 0.5f, Waves.TRIANGLE );
  waveG.patch( out );
  waveB = new Oscil( 0, 0.5f, Waves.SINE );
  waveB.patch( out );
}
void draw() {
  
  if(mils != millis()/(int)speed && j<frequenciesR.length)
  { 
    float amp = map( mouseY, 0, height, 1, 0 );
    speed = map(mouseX, 0, width, 1, 100);
    waveR.setAmplitude( amp/2 );
    waveG.setAmplitude( amp );
    waveB.setAmplitude( amp );
    float freq = frequenciesR[j];
    waveR.setFrequency( freq );
    freq = frequenciesG[j];
    waveG.setFrequency( freq*2 );
    freq = frequenciesB[j];
    waveB.setFrequency( freq*4 );
    image.pixels[j] = color(255,255,255);
    image.updatePixels(); 
    j++;
    mils = millis()/(int)speed;
  }
  image(image, 0, 0);
}

var NN = {};

(function(global) {
  "use strict";
  
  // Utility fun
  function assert(condition, message) {
    if (!condition) {
      message = message || "Assertion failed";
      if (typeof Error !== "undefined") {
        throw new Error(message);
      }
      throw message; // Fallback
    }
  }
  
  var layer = function(p_amount, l_id)
  {
    this.layerID = l_id;
    this.numberOfPerceptrons = p_amount;
    this.perceptrons = [];
    this.weightMatrix = [];
    return this;
  }
  
  var connection = function(p_sender, c_weight, p_reciever)
  {
    this.sender = p_sender;
    this.weight = c_weight;
    this.reciever = p_reciever;
    return this;
  }
  
  var perceptron = function(p_id, l_id)
  {
    this.id = p_id;
    this.layerID = l_id;
    this.connections = [];
    this.inpt = 0;
    this.outpt = 0;
    return this;
  }
  
  var neuralNet = function(layers)
  {
    this.end = layers.length-1;
    this.inputLayer = layers[0];
    this.outputLayer = layers[this.end];
    this.hiddenLayers = layers;
    this.hiddenLayers.splice(0,1);
    this.hiddenLayers.splice(this.end-1,1);
    
    this.fullLayers = [];
    
    this.buildNetwork();
  }
  neuralNet.prototype =
  {
    buildNetwork : function()
    {
      for(var h=0; h<this.inputLayer.numberOfPerceptrons; h++)
      {
        var i_prcptrn = new perceptron(h, 0);
        this.inputLayer.perceptrons.push(i_prcptrn);
      }
      this.fullLayers.push(this.inputLayer);
      
      for(var i=0; i<this.hiddenLayers.length; i++)
      {
        for(var j=0; j<this.hiddenLayers[i].numberOfPerceptrons; j++)
        {
          var h_prcptrn = new perceptron(j, i+1);
          this.hiddenLayers[i].perceptrons.push(h_prcptrn);
        }
        this.fullLayers.push(this.hiddenLayers[i]);
      }
      for(var k=0; k<this.outputLayer.numberOfPerceptrons; k++)
      {
        var o_prcptrn = new perceptron(k, this.end);
        this.outputLayer.perceptrons.push(o_prcptrn);
      }
      this.fullLayers.push(this.outputLayer);
      
      for(var l=0; l<this.fullLayers.length; l++)
      {
        if(l < this.fullLayers.length-1)
        {
          this.fullLayers[l].id = l;
          this.fullLayers[l+1].id = l+1;
          this.makeConnections(this.fullLayers[l], this.fullLayers[l+1]);
        }
      }
    },
    
    makeConnections : function (sendLayer, recieveLayer)
    {
      var senderLayer = sendLayer;
      var recieverLayer = recieveLayer;
      
      for(var i=0; i<recieverLayer.perceptrons.length; i++)
      {
        for(var j=0; j<senderLayer.perceptrons.length; j++)
        {
          if(recieverLayer.id != senderLayer.id)
          {
            var cnnctn = new connection(senderLayer.perceptrons[j], randf(-1,1), recieverLayer.perceptrons[i]);
            recieverLayer.perceptrons[i].connections.push(cnnctn);
          }
        }
      }
    },
    
    saveWeightMatrix : function (layer)
    {
      
    },
    
    feedNetwork : function (inputs)
    {
      var inputList = inputs;
      var inp = 0;
      for(var i=0; i<inputs.length; i++)
      {
        this.inputLayer.perceptrons[i].inpt = inputList[i];
      }
      for(var j=0; j<this.hiddenLayers.length; j++)
      {
        for(var k=0; k<this.hiddenLayers[j].perceptrons.length; k++)
        {
          for(var l=0; l<this.hiddenLayers[j].perceptrons[k].connections.length; l++)
          {
            var si = this.hiddenLayers[j].perceptrons[k].connections[l].sender.inpt;
            var cw = this.hiddenLayers[j].perceptrons[k].connections[l].weight;
            inp +=  si * cw;
            this.hiddenLayers[j].perceptrons[k].inpt = inp;
          }
          this.hiddenLayers[j].perceptrons[k].outpt = sig(this.hiddenLayers[j].perceptrons[k].inpt);
        }
      }
      inp = 0;
      for(var n=0; n<this.outputLayer.perceptrons.length; n++)
      {
        for(var m=0; m<this.outputLayer.perceptrons[n].connections.length; m++)
        {
          var si = this.outputLayer.perceptrons[n].connections[m].sender.inpt;
          var cw = this.outputLayer.perceptrons[n].connections[m].weight;
          inp +=  si * cw;
          this.outputLayer.perceptrons[n].inpt = inp;
        }
        this.outputLayer.perceptrons[n].outpt = sig(this.outputLayer.perceptrons[n].inpt);
      }
    }
    
  }
  //random
  var randf = function(a, b) 
  { 
    return Math.random()*(b-a)+a;
  }
  //activation
  var sig = function(t)
  {
    return 1/(1+Math.pow(Math.E, -t));
  }
  var tanh = function(t)
  {
    return (Math.exp(t) - Math.exp(-t)) / (Math.exp(t) + Math.exp(-t));
  }
  
  //acces
  global.layer = layer;
  global.connection = connection;
  global.perceptron = perceptron;
  global.neuralNet = neuralNet;
  global.assert = assert;
  
  global.ranf = randf;

})(NN);

var audioClip;
var fft;
var spectrum;
var waveform;
var currentImg;
var images = [];

function preload()
{
  audioClip = loadSound('Audio/LangSon.mp3');
}

var layers = [];
var input;
var inputs = [];
var bias = 1;
var nn;
var sizeh = 256;
var sizew = 512;

var img = function()
{
  var newImg = createImage(sizew,sizeh);
  images.push(newImg);
  return newImg;
}
function setup()
{
  input = new NN.layer(3);
  layers.push(input);
  var hidden = new NN.layer(5);
  layers.push(hidden);
  var output = new NN.layer(3);
  layers.push(output);
  
  nn = new NN.neuralNet(layers);
  
  createCanvas(sizew*5, sizeh*15);
  currentImg = img();

  if(audioClip)
  {
    audioClip.play();
  }
  fft = new p5.FFT();
}

var r,g,b,w,f,k,l=0;
var outColor;

function buildImg(x,y,c)
{
  currentImg.loadPixels();
  currentImg.set(x, y, c);
}

var x = 0;
function draw()
{
  spectrum = fft.analyze();
  waveform = fft.waveform();
  if(l==sizeh)
  {
    l=0;
    x+=sizeh;
    currentImg = img();
  }
  currentImg.loadPixels();
  for (k = 0; k<waveform.length; k++)
  {
      w = map(waveform[k], -1, 1, -4, 4);
      f =  map(spectrum[k], 0, 255, -4, 4);
      inputs = [bias,w,f];
      nn.feedNetwork(inputs);
      r = nn.outputLayer.perceptrons[0].outpt*255.0;
      g = nn.outputLayer.perceptrons[1].outpt*255.0;
      b = nn.outputLayer.perceptrons[2].outpt*255.0;
      outColor = color(r,g,b);
      currentImg.set(k, l, outColor);
  }
  l++;
 
  currentImg.updatePixels();
  image(currentImg, 0, x);
}