christian faubel

semester plan historical overview: turing von neumann mc’ culloch & pitts rosenblatt (perceptron) marvin minsky (xor) Kohonen & von der Malsburg Connectionism Mc’lelland neocognitron goodiepal: grey walter braitenberg stumpy The Perceptron in order to understand the euphoria when Frank Rosenblatt presented the Perceptron and especially the learning algorithm one has to be aware of fundamental importance of propositional logic in mathematics but also in philosophy. There is quite a good online platform in german for learning propositional logic called Mathe für Nicht-Freaks, here we just want get a basic understanding of common logical operations on propositions. To that end we will program a simple p5js -script to play around with the basic logical operations such as AND, OR, XOR, etc.

x_1=[0,1,0,1];
x_2=[0,0,1,1];

function setup() {
  createCanvas(400,400);  
}

function draw() {
  colorMode(HSB,360,1,1,1);
  strokeWeight(0);
  for (i=0;i<4;i++)
  {
    fill(0,1,0.3+x_1[i]);
    ellipse(100,50+75*i,50,50)
    fill(0,1,0.3+x_2[i]);
    ellipse(175,50+75*i,50,50)
    fill(0,1,0.3+ (x_1[i] | x_2[i]));
    ellipse(275,50+75*i,50,50)    
  }
}

the perceptron learning rule

The perceptron and the perceptron learning rule was introduced by Frank Rosenblatt in 1957. A single perceptron is most similar to an integrate and fire neuron without temporal dynamics. It can have a number of inputs which are multiplied by some weights and the all integrated (summed). It has an internal bias term \(b \) that shifts the threshold of an all or nothing threshold function. $$output = f_{thres}(w_{1}*i_{1} + w_{2}*i_{2} + b) $$ The bias term can be replaced with an input clamped to the value of one and a weight that takes the value of the bias. For example a negative weight of – 0.5 would then correspond to a bias term of -0.5. To start we will realize the logical AND function with a perceptron. The perceptron will have three inputs, the first \(i_{0}\) for the bias term, the second \(i_{1}\) for the logical expression A and the third \(i_{2}\) for the logical expression B. This task ist straightforward, setting weight for the bias term to zero \(w_{0}=-1\), and the two other weights to one \(w_{1}=1,w_{2}=1\). $$output = f_{thres}(1*i_{1} + 1*i_{2} + -1*1) $$

x_1 = [0, 1, 0, 1];
x_2 = [0, 0, 1, 1];
x_0 = [1, 1, 1, 1];

y = [0, 0, 0, 0];
w_0 = -1;
w_1 = 1;
w_2 = 1;

function setup() {
  cnv=createCanvas(400, 400);
  cnv.parent("proplogic");
}

function draw() {
  colorMode(HSB, 360, 1, 1, 1);
  strokeWeight(0);
  for (i = 0; i < 4; i++) { fill(0, 1, 0.3 + x_1[i]); ellipse(100, 50 + 75 * i, 50, 50) fill(0, 1, 0.3 + x_2[i]); ellipse(175, 50 + 75 * i, 50, 50) y[i] = threshold(x_0[i] * w_0 + x_1[i] * w_1 + x_2[i] * w_2); fill(0,1,0.3+ y[i]); ellipse(275,50+75*i,50,50) } } function threshold(input) { if (input > 0)
    return 1;
  else
    return 0
}

Learning with the perceptron means to adjust the weights automatically according to some rule so that the perceptron will produce a specific output for a given input. The rule that has been proposed by Frank Rosenblatt uses the error, which is the deviation of the perceptron’s output from the expected output as correctiv signal. For the AND function the expected output \(y\) is [0,0,0,1] given the inputs A [0,0,1,1] and B [0,1,0,1]. The error \(e\) is thus formulated as the difference of the expected output and the output computed by the perceptron. $$ e = y -output $$ This error term is then multiplied with a learning factor and with the specific input combination that produced the error to produce an update term \( \Delta w_{i}\) $$\Delta w_{i}= \lambda \times e \times x_{i} $$ The corresponding weight \(w_{i}\) is then updated by integrating this update term \(\Delta w_{i}\) $$w_{i,new}=w_{i,old} + \Delta w_{i}$$computing the learning of the AND function “by foot” slides.

//define input variable
// bracket [alt 5 de-mac] signals that x will be a vector/Matrix
var x = [];
//define network weights
var w = [];
//define the variation of the weights and initialise to 0
var d_w = [0, 0, 0];
//define the error variable and initialize to 0
var error = 0;
//define and initialize the expected output 
//definition of the AND function
var y = [0, 1, 1, 1];
//fix the learn_rate, this a parameter to experiment with
var learn_rate = 0.1;
//counter variable that will incremented on every call of draw 
var ctr = 0;
//curly bracket defines a code block {alt 8 de-mac}
function setup() {
  frameRate(10);
  createCanvas(400, 500);
  //initialise the entry with index 0 as vector 
  // this makes x a 2 dimensional matrix
  x[0] = [];
  //initialise all values in column 0, let j run from 0 to 4
  for (j = 0; j < 4; j++) {
    x[0][j] = 1;
  }
  //initialise the entry with index 1 as vector
  x[1] = [];
  for (j = 0; j < 4; j++) { //divide j by two and only keep the integer value -> creates 0,0,1,1
    x[1][j] = int(j / 2);
  }
  //initialise the entry with index 2 as vector  
  x[2] = [];
  for (j = 0; j < 4; j++) {
    x[2][j] = j % 2;
  }
  //initialise the weight vector
  w = [0, 0, 0];
}

function draw() {
  //set colormode to hue saturation brightness and alpha with hue from 0-360
  colorMode(HSB, 360, 1, 1, 1);
  //set background to white  saturation=0, brighness =1
  background(0, 0, 1);
  //increment the counter
  ctr = ctr + 1;
  //every call loop through the 4 differnt input input combination with a modulo 4 operation 
  var index = ctr % 4;
  fill(200, 0, 0);
  textSize(20);
  text(ctr + " " + index, 200, 50);
  //here is the core computation of the perceptron, weight 0 is multiplied with input 0 .
  //weight 1 with input 1, weight 2 at input 2 all at position index 
  var perceptron_output = threshold(w[0] * x[0][index] + w[1] * x[1][index] + w[2] * x[2][index]);
  //the error is the differnce between the expected output in y[index] and the output
  //from the perceptron
  error = y[index] - perceptron_output;
  i = 0;
  while (i < 3) {
    //the variation of the weigth i 
    //is the error times the learning rate times the input i at index i at
    d_w[i] = error * learn_rate * x[i][index];
    w[i] = w[i] + d_w[i];
    i = i + 1;
  }
  textSize(30);
  fill(20, 0, 0);
  text("error: " + error, 200, 100);
  i = 0;
  while (i < 3) { fill(20, 1, x[i][index]); ellipse(50, 50 + i * 50, 40, 40); i = i + 1; } fill(20, 1, perceptron_output); ellipse(100, 100, 40, 40); } //the threshold function mimiking the behavior of a neuron to only fire when activated //above a certain level defined by x function threshold(x) { if (x > 0) {
    return 1;
  } else //if x is not > 0 than it is <= 0
  {
    return 0;
  }
}

//define variable as array
//variables for display
var a_img = [];
var b_img = [];
var w_img = [];

//input variable for the perceptron 
var input = [];
//expected output for the perecptron (A ->1 ) (B ->0)
var expected_output = [1, 0];
//weights of the perceptron
var w = [];

//global varviable gridsize
var grid_size = 10;

//global counter variable
var ctr = 0

//program flow control variables
run = 0;
test = 0;
var use_letter = 0;

//learn_Rate
var learn_rate = 0.2;

function setup() {
  //initialize input vector that will take values from input
  // images a_img and b_img
  input[0] = [];
  input[1] = [];
  var i = 1;
  //set first input value to 1 -> trains the bias term
  input[0][0] = 1;
  input[1][0] = 1;
  //set first weight to 0;
  w[0] = 0;
  //initialize image matrices
  for (var x = 0; x < 20; x++) {
    //define second dimension
    a_img[x] = [];
    b_img[x] = [];
    w_img[x] = [];
    //run through each row and fill the values 
    for (var y = 0; y < 20; y++) {
      a_img[x][y] = 0;
      input[0][i] = a_img[x][y];
      b_img[x][y] = 0;
      input[1][i] = b_img[x][y];
      w_img[x][y] = 0 * random(0, 1);
      w[i] = w_img[x][y];
      i = i + 1;
    }
  }
  frameRate(6);
  createCanvas(400, 400);
}

function draw() {
  ctr = ctr + 1;
  background(255);
  //draw letter at the topmost left corner with gridsize 10
  drawLetter(0, 0, grid_size, 200, 200, a_img);
  //draw second letter just below
  drawLetter(0, 200, grid_size, 200, 400, b_img);
  //draw weights to the right of the second image
  drawLetter(200, 200, grid_size, 400, 400, w_img);
  var sum = 0;
  var letter_index = ctr % 2;
  //from here on the perceptron is computed 
  //sum all inputs multiplied with their weights x0*w0+x1*w1 ...
  sum = 0;
  for (var i = 0; i < 401; i++) {
    sum = sum + input[letter_index][i] * w[i];
  }
  //the perceptrons output is the threshold of all weights times their corresponding inputs
  var perceptron_output = threshold(sum);
  //compute the error
  //the error is the differnce between the expected output in y[index] and the output
  //from the perceptron
  //only update perceptrons weights if run is active
  if (run == 1) {
    error = expected_output[letter_index] - perceptron_output;
    for (var i = 0; i < 401; i++) {
      //the variation of the weigth i 
      //is the error times the learning rate times the input i at index i at
      var d_w = error * learn_rate * input[letter_index][i];
      w[i] = w[i] + d_w;
    }
  }
  //no more learning just computing the perceptron output without modifying the weights
  if (test == 1) {
    //sum all inputs multiplied with their weights x0*w0+x1*w1 ...
    sum = 0;
    for (var i = 0; i < 401; i++) {
      sum = sum + input[use_letter][i] * w[i];
    }
    //the perceptrons output is the threshold of all weights times their corresponding inputs
    var perceptron_output = threshold(sum);
    if (perceptron_output == 1) {
      fill(0);
      textSize(150);
      text("A",width/4+width / 2-40, height / 4+40);
    } else {
      fill(0);
      textSize(100);
      text("B",width/4+width / 2, height / 4);
    }
    if (use_letter==0)
    {
       //draw rectangle around the chosen inout letter
      noFill();
      stroke(255,0,0);
      strokeWeight(2);
      rect(0,0,width/2,height/2);
    }
    else
    {
        //draw rectangle around the chosen inout letter
       noFill();
      stroke(255,0,0);
      strokeWeight(2);
      rect(0,height/2,width/2,height/2);
    }
  }
  strokeWeight(1);
  //visualize the weights as 2dim image
  var i = 1;
  for (var x = 0; x < 20; x++) {
    for (var y = 0; y < 20; y++) {
      w_img[x][y] = (w[i] + 0.5) * 1;
      i = i + 1;
    }
  }
}
// this function is used to paint the 2 dimensional variable containing the
// image of the letter, start_x and start_y define the top left corner
// gridsize stes the resolution and and end_x and end_y define the right 
// bottom corner
function drawLetter(start_x, start_y, grid_size, end_x, end_y, img) {
  for (var x = 0; x < (end_x - start_x) / grid_size; x++) {
    for (var y = 0; y < (end_y - start_y) / grid_size; y++) {
      //set fill value to value of the image
      stroke(0);
      fill((1 - img[x][y]) * 255);
      rect(start_x + x * grid_size, start_y + y * grid_size, grid_size, grid_size);
    }
  }
}
//this function is used to draw letetrs with the mouse
function mouseDragged() {
  //if the mouse is positioned in  the left top square the first letter A is drawn
  if ((mouseX < width / 2) && (mouseY < height / 2)) {
    var index_x = int(mouseX / grid_size);
    var index_y = int(mouseY / grid_size);
    a_img[index_x][index_y] = 1;
    input[0][index_x * 20 + index_y] = 1;
  }
  //if the mouse is positioned in the left bottom square the second letter B is drawn
  if ((mouseX < width / 2) && (mouseY > height / 2)) {
    var index_x = int(mouseX / grid_size);
    var index_y = int((mouseY - height / 2) / grid_size);
    b_img[index_x][index_y] = 1;
    input[1][index_x * 20 + index_y] = 1;
  }
}

function keyPressed() {
  print(keyCode);
  //x-key ->reset all images
  if (keyCode == 88) {
    for (var x = 0; x < 20; x++) {
      for (var y = 0; y < 20; y++) {
        a_img[x][y] = 0;
        b_img[x][y] = 0;
        input[0][x * 20 + y] = 0;
        input[1][x * 20 + y] = 0;
      }
    }
  }
  //r-key run learning
  if (keyCode == 82) {
    run = 1;
    test = 0;
  }
  //t-key testing
  if (keyCode == 84) {
    test = 1;
    run = 0;
  }
  //a-key sets letter index to 0
  if (keyCode == 66) {
    use_letter =  1;
  }
  //b key sets letter index to 1
  if (keyCode == 65) {
    use_letter = 0;
  }
}

function threshold(x) {
  if (x > 0)
    return 1;
  else
    return 0;
}

the drawing machine, for connecting the perceptron to action, idea to use this vertical plotter for the seminar and all technical preparation: vincent brinkmann