topical media & game development
actionscript-omr-console.mx
actionscript-omr-console.mx
[swf]
flex
<?xml version="1.0" encoding="utf-8"?>
<mx:Application xmlns:mx="http://www.adobe.com/2006/mxml" xmlns:MyComp="*" layout="absolute">
<MyComp:actionscript_omr_lib_WebcamPanel x="20" y="22" width="700" height="550" layout="absolute" title="Camera Stream" id="pnlWebcam" >
</MyComp:actionscript_omr_lib_WebcamPanel>
<mx:Panel x="780" y="580" width="700" height="550" layout="absolute" title="Notes" id="pnlSnapshot" >
</mx:Panel>
<mx:Panel x="780" y="22" width="700" height="550" layout="absolute" title="Interpretation" id="pnlInterpretation" >
</mx:Panel>
<mx:HSlider id="thresholdSlider" snapInterval="1" value="5" maximum="60" thumbDrag="updateThreshold(event);" liveDragging="true" height="20" left="396" top="0" width="200"/>
<MyComp:actionscript_omr_lib_DragPanel width="300" height="200" x="20" y="20" title="Help" visible="false" id = "dragPanel"/>
<mx:Panel x="20" y="580" id="console" title="console" maxWidth="700" height="360" layout="absolute" horizontalScrollPolicy="off">
<mx:Panel layout="horizontal">
<mx:Button label="Take Snapshot" click="takeSnapshot()"/>
<mx:Button label="Apply Convolution Filter" click="applyFilter()"/>
<mx:Button label="Draw Interpreted Lines" click="detectLines()"/>
<mx:Button label="Interpret Clips" click="interpretClips()"/>
<mx:Label text="position" id="counterlabel"/>
</mx:Panel>
<mx:TextArea y="70" id="consolelabel" text="" editable="false" wordWrap="true" width="660" height="170"/>
</mx:Panel>
<mx:HBox>
<mx:Button label="Help" click="showHelp();"/>
<mx:Button label="play" click="playSound();"/>
<mx:Button label="stop" click="stopSound();"/>
</mx:HBox>
<mx:Script >
<![CDATA[
import mx.controls.Text;
import mx.events.ItemClickEvent;
import mx.controls.Button;
import mx.controls.Label;
import mx.core.UIComponent;
import flash.display.BitmapData;
import mx.core.SoundAsset;
import flash.media.*;
//import myComponents.*;
import mx.controls.Alert;
private var threshold:int = 0;
private var audio:SoundChannel = new SoundChannel();
private var glyph:actionscript_omr_lib_Glyph = new actionscript_omr_lib_Glyph();
private var music:actionscript_omr_lib_Music = new actionscript_omr_lib_Music();
private var globalSnapshot:Bitmap;
private var currentStaff:actionscript_omr_lib_Staff;
private var helperData:BitmapData;
private var helperTransform:ColorTransform;
private var clips_matrix:Array;
private var clips_array:Array;
private var playBack:String = "piano";
private function playSound():void{
Play back the interpreted notes as sinewaves or piano samples*
if(playBack=="wave"){
currentStaff.play();
}
if(playBack=="piano"){
currentStaff.playPiano();
}
}
private function stopSound():void{
Stop playback*
if(playBack=="wave"){
audio.stop();}
if(playBack=="piano"){
//No way to stop piano sequence without messing up the queue
}
}
public function updateThreshold(event:Event):void{
Update the noise threshold from slider event*
threshold=thresholdSlider.value;}
public function interpretClips():void{
Interprets Clips from the sampled bitmap in clips_matrix
//Create music structure object
music = new actionscript_omr_lib_Music();
var rowflag:Boolean=false;
var lineMap:Array = new Array();
for (var x:int =0;x<=clips_matrix.length;x++) {
var rowcounter:int = 0;
for each (var clip:actionscript_omr_lib_PictureElement in clips_matrix[x]){
if (clip.isLine()) rowcounter++;
}
//Are more than half of the row's elements a potential line? Then it's probably a line.
if(rowcounter>(clips_matrix[0].length/2)){
lineMap.push(x);
}
}
//lineMap now contains all the rows that are probably lines
//Unless the camera is too far away, there should be at least 2 or 3 rows for a staff
var lineFlag:Boolean=true;
var lineScale:int =1;
var lineScalehigh:int=0;
for (var x:int = 0;x<lineMap.length;x++){
//The following check is for failure in consecutive staff
if(lineFlag){
if(lineMap[x+1]-lineMap[x]==1){
if(lineScale>lineScalehigh)lineScalehigh=lineScale;
lineScale++;
} else if(lineScale>1){
lineFlag=false;
}
}
// printConsole("Row " +lineMap[x]+" is part of a staff\n");
}
// printConsole("Staff is covered by " +lineScale+" picture elements\n");
var staffFoundFlag:Boolean = false;
//Determine the scale
for (var x:int = 0;x<lineMap.length;x++){
if(lineMap[x+1]-lineMap[x]==1){
// printConsole("two in a row found on " +lineMap[x]+"\n");
if(lineMap[x+lineScale-1]-lineMap[x]==lineScale-1){
printConsole("scale measure lines up on " +lineMap[x]+", pos " +x+" in the line map of length "+lineMap.length+" fc " +(lineMap[x+lineScale-1])+ "-"+lineMap[x]+"=="+(lineScale-1));
music.addStaff(lineMap[x],lineScale);
}
}
}
for each (var staff:actionscript_omr_lib_Staff in music.getStaffs()){
printConsole("There is a staff on row "+staff.getRow()+"with scale" +staff.getScale());
clips_matrix[staff.getRow()][0].y;
//Create the staff interpretation button to the right of the interpretation panel
var staffSelector:Button = new Button();
staffSelector.addEventListener(
MouseEvent.MOUSE_DOWN,
function(evt:MouseEvent):void {
//This function can't inherit ANY of the local variables. Making an event is not supported, TODO!!
var passStaff:actionscript_omr_lib_Staff = staff;
interpretStaff(passStaff);
}
);
printConsole("added event listener for row" +staff.getRow());
staffSelector.width=10;
staffSelector.height=clips_matrix[0][0].getRect(this).height*staff.getScale();
staffSelector.x=650;
staffSelector.y=clips_matrix[staff.getRow()][0].y;
pnlInterpretation.addChild(staffSelector);
}
}
private function interpretStaff(discard:actionscript_omr_lib_Staff){
//Take the first staff by default, if a subsequent staff is selected via the staffSelector, the staff interpretation is messed up
var staff:actionscript_omr_lib_Staff = music.getStaffs()[0];
printConsole("interpreting Staff on row "+staff.getRow());
var scale:int = staff.getScale();
var staffy:int = clips_matrix[staff.getRow()][0].y-(clips_matrix[staff.getRow()][0].height)*(scale); //Add scale's worth of rows
var staffheight:int = clips_matrix[staff.getRow()][0].height*(scale*3)
//create a rectangle from interpreted staff bounds
var rect:Rectangle = new Rectangle(0,staffy,globalSnapshot.width, staffheight);
var staffBitmapData:BitmapData = new BitmapData(rect.width,rect.height,true,0xFFFFFFFF);
staffBitmapData.copyPixels(globalSnapshot.bitmapData,rect,new Point(0,0));
var pt:Point = new Point(0, 0);
var rect2:Rectangle = new Rectangle(0,0,rect.width,rect.height);
var thresholds:uint = (threshold*(16*16))+threshold*(16*16*16*16)+threshold+4278190080;
var color:uint = 0x00000000;
var maskColor:uint = 0xFFFFFFFF;
//'Zoom in' on the staff and get ready for another detection process
var pixelCount:int=staffBitmapData.threshold(staffBitmapData, rect2, pt, ">", thresholds, color, maskColor, false);
var staffBitmap:Bitmap = new Bitmap(staffBitmapData);
var staffHolder:UIComponent = new UIComponent();
staffHolder.addChild(staffBitmap);
var diameter:int = (clips_matrix[0][0].height)*scale/4;
var radius:int = diameter/2;
//a note is generally 1/4th the height of the staff (B step is center)
var multiplier:int = 8;
//double diameter for checks
//fiddle with this
staffBitmapData = removeWholeLines(staffBitmapData,scale);
var result:BitmapData = new BitmapData(diameter+multiplier,diameter+multiplier);//(diameter,
// diameter,
// false,
// 0x808080);
//this approach is VERY brute force
for(var y:int = 0; y<staffBitmapData.height;y+=diameter){
for(var x:int = 0; x< staffBitmapData.width;x+=diameter){
var noteRect:Rectangle = new Rectangle(x,y,diameter+multiplier,diameter+multiplier);
result.copyPixels(staffBitmapData,noteRect,new Point(0,0));
// Loop through the pixels in the image one by one
//checkNote(diameter,x,y,result,)
var insideCounter:int =0;
var outsideCounter:int=0;
//Start feature detection on curvature with blobs
for (var i:int = 0; i < diameter+multiplier; i++)
{
for (var j:int = 0; j < diameter+multiplier; j++)
{
// Calculate the x and y distances of this pixel from
// the center of the circle (as a percentage of the radius).
var pctX:Number = (i+(multiplier/2) - radius) / radius;
var pctY:Number = (j+(multiplier/2) - radius) / radius;
// Calculate the linear distance of this pixel from
// the center of the circle (as a percentage of the radius).
var pctDistance:Number = Math.sqrt(pctX * pctX + pctY * pctY);
// If the current pixel is inside the circle,
// see if it's on
if (pctDistance < 1)
{
if(result.getPixel32(i+(multiplier/2),j+(multiplier/2)) == 0x00000000){
// printConsole("counted white pixel inside at "+i+","+j+" in "+x+","+y);
insideCounter++
}
}
//if the current pixel is outside of the circle, see if it's on
else{
if(result.getPixel32(i+(multiplier/2),j+(multiplier/2)) == 0x00000000){
outsideCounter++;
// printConsole("counted white pixel outside at "+i+","+j+" in "+x+","+y);
}
}
}
}
if (insideCounter/outsideCounter>0.25){
//Add the note to the staff object
//clip height * (scale/4) is the magic scaling distance (4 spaces between staff lines)
//Note : For semitone scaling, multiply the magic distance by 2, this decreases accuracy by a lot but allows semitones
var notePitch:int = y/(clips_matrix[0][0].height*(scale/4));
printConsole ("notePitch is "+notePitch);
staff.addNote(4,notePitch);
printConsole("this circle may be a note, it's at "+x+","+y+" with insidecounter at "+insideCounter+" and outside at "+outsideCounter);
//following code draws copies of detected notes on top of the original webcam feed, but doesn't line up with the original image due to size constraints
//this dummpyholder is never used
var tempBitmap:Bitmap = new Bitmap(result.clone());
var tempSprite:Sprite = new Sprite();
tempSprite.addChild(tempBitmap);
tempSprite.x=x+diameter+multiplier/2;
tempSprite.y=y+diameter+multiplier/2;
var tempHolder:UIComponent = new UIComponent();
tempHolder.addChild(tempSprite);
//pnlWebcam.addChild(tempHolder);
var circle:Shape = new Shape();
var xPos:Number = multiplier/2+x+diameter/2;
var yPos:Number = multiplier/2+y+diameter/2;
var radius2:Number = diameter/2;
circle.graphics.beginFill(0x20FF8800);
circle.graphics.drawCircle(xPos, yPos, radius2);
staffHolder.addChild(circle);
}
}
}
pnlSnapshot.addChild(staffHolder);
currentStaff = staff;
}
public function detectLines():void{
Takes a snapshot from the camera feed and samples the bitmap
var bitmap:Bitmap = takeSnapshot();
globalSnapshot = bitmap;
//Apply convolution filter
var bitmapdata:BitmapData = applyFilterToBitmapData(bitmap.bitmapData);
var interpretbitmap:Bitmap = new Bitmap(bitmapdata);
searchGlyphs(interpretbitmap,threshold);
}
public function searchGlyphs(interpretbitmap:Bitmap,thresholdparam:int):void{
var data:BitmapData = interpretbitmap.bitmapData;
clips_matrix = new Array();
clips_array = new Array();
var rows:int=20;
var columns:int=20;
var ww:int=Math.ceil(interpretbitmap.width/rows);
var hh:int=Math.ceil(interpretbitmap.height/columns);
var mat:Matrix;
var bitmap_data:BitmapData;
var point:Point;
var clip:actionscript_omr_lib_PictureElement;
var interpretHolder:UIComponent = new UIComponent();
for(var i:int=0;i < columns;i++)
{
clips_matrix[i] = new Array();
for(var j:int=0;j < rows;j++)
{
bitmap_data=new BitmapData(ww,hh,true,0xFFFFFFFF);
mat=interpretbitmap.transform.matrix;
mat.translate(-ww*j,-hh*i);
bitmap_data.draw(interpretbitmap,mat);
clip=new actionscript_omr_lib_PictureElement();
clips_array.push(clip);
interpretHolder.addChild(clip);
//uncomment next line to show white lines around clips, good for seeing sample size
// point=new Point(interpretbitmap.x+ww*j+1*j,interpretbitmap.y+hh*i+1*i);
point=new Point(interpretbitmap.x+ww*j,interpretbitmap.y+hh*i);
var bitmap:Bitmap=new Bitmap(bitmap_data);
var pt:Point = new Point(0, 0);
var rect:Rectangle = new Rectangle(0, 0, ww, hh);
var threshold:uint = (thresholdparam*(16*16))+thresholdparam*(16*16*16*16)+thresholdparam+4278190080;
var color:uint = 0x00000000;
var maskColor:uint = 0xFFFFFFF;
var pixelCount:int=bitmap_data.threshold(bitmap_data, rect, pt, ">", threshold, color, maskColor, false);
//Count pixels, if there are more than thirty over the threshold, the clip is relevant
if(pixelCount>30){
//todo If the whole screen is 'musically relevant' there is no paper
//check for staff lines
if(checkForLines(bitmap_data)){
//Add green overlay for visual presentation
var overlay_data3:BitmapData = new BitmapData(ww,hh,true,0xFF00FF00);
clip.addChild(new Bitmap(overlay_data3));
//Tell clip it's a line
clip.setLine();
}
//compare to bitmap for glyp search -shouldn't be else clause-
//TODO not currently using this
//else if(compareBitmaps(new Bitmap(bitmap_data),glyph.getQuaver())>20){
//Add blue overlay for visual presentation
// var overlay_data2:BitmapData = new BitmapData(ww,hh,true,0xFF0000FF);
// clip.addChild(new Bitmap(overlay_data2));
//Tell clip it's a note
// clip.setNote();
//}
else{
//Nothing fancy about this potential
//Add red overlay for visual presentation
var overlay_data:BitmapData = new BitmapData(ww,hh,true,0xFFFF0000);
clip.addChild(new Bitmap(overlay_data));
}
}
//add the clip
clip.addChild(bitmap);
clip.x=point.x;
clip.y=point.y;
//add the clip to clips matrix
clips_matrix[i][j] = clip;
//end for loop j
}
//end for loop i
}
//Add the overlays and clips to Interpretation panel
pnlInterpretation.removeAllChildren();
pnlInterpretation.addChild(interpretHolder);
}
public function removeWholeLines(bitmap:BitmapData,scale:int):BitmapData{
//removes whole lines from bitmap
for(var x:int = 0;x<=bitmap.height;x++){
//do in chunks of 10 ---arbitrary number, make variable
//for(var y:int = 0;y<=bitmap.height/5;y++){
//
//}
var rect:Rectangle = new Rectangle(0,x,bitmap.width,1);
var array1:ByteArray = bitmap.getPixels(rect);
array1.position=0;
var flag:Boolean=false;
var counter:int=0;
var highcounter:int=0;
var linelimit:int = scale*3;
while(array1.bytesAvailable){
if(flag){counter++;
if(counter>linelimit){
array1.position=0;
while(array1.bytesAvailable){
array1.writeUnsignedInt(0xFF000000);
}
}
}
if(array1.bytesAvailable){
if(array1.readUnsignedInt()==0x00000000){
flag=true
}
else{
flag=false;
if(highcounter<counter)highcounter=counter;
counter=0;
}
}
}
array1.position=0;
bitmap.setPixels(rect,array1);
//if(highcounter>minimum){}
}
return bitmap;
}
public function removeLines(bitmap:BitmapData):BitmapData{
//removes lines from bitmap
for(var x:int = 0;x<=bitmap.height;x++){
//do in chunks of 10 ---arbitrary number, make variable
//for(var y:int = 0;y<=bitmap.height/5;y++){
//
//}
var rect:Rectangle = new Rectangle(0,x,bitmap.width,1);
var array1:ByteArray = bitmap.getPixels(rect);
array1.position=0;
var flag:Boolean=false;
var counter:int=0;
var highcounter:int=0;
var linelimit = 20;
while(array1.bytesAvailable){
if(flag){counter++;
if(counter>linelimit){
array1.position-=linelimit*4; //times four, Uint is 32 bytes
for(var x:int = 0; x<linelimit;x++){
array1.writeUnsignedInt(0xFF000000);
}
}
}
if(array1.readUnsignedInt()==0x00000000){
flag=true
}
else{
flag=false;
if(highcounter<counter)highcounter=counter;
counter=0;
}
}
array1.position=0;
bitmap.setPixels(rect,array1);
//if(highcounter>minimum){}
}
return bitmap;
}
public function checkForLines(bitmap:BitmapData):Boolean{
//Returns >0 score if an uninterrupted line longer than minimum was found
//create scanlines
//TODO something wrong with the first line scanned
var minimum:int = 10;
var score:int = 0;
for(var x:int = 0;x<=bitmap.height;x++){
var rect:Rectangle = new Rectangle(0,x,bitmap.width,1);
var array1:ByteArray = bitmap.getPixels(rect);
array1.position=0;
var flag:Boolean=false;
var counter:int=0;
var highcounter:int=0;
while(array1.bytesAvailable){
//flag never true, color matched probably incorrect !!!!!!!!!!!!!!!!!!!!!!!!!!!!<<<<------------ na voetbal
if(flag){counter++;}
if(array1.readUnsignedInt()==0x00000000){
flag=true
}
else{
flag=false;
if(highcounter<counter)highcounter=counter;
counter=0;
}
}
if(highcounter>minimum)score++
}
return score>0;
}
public function compareBitmaps(bitmap1:Bitmap,bitmap2:Bitmap):int{
//cue bitmap comparison algorithm, I wish flex had this built in
if(bitmap1.height != bitmap2.height || bitmap1.width != bitmap2.width){
//TODO : Scale
printConsole("bitmaps are not of equal size");
return 0;
}
var score:int=0;
var rect:Rectangle = new Rectangle(0,0,bitmap1.width,bitmap1.height)
var array1:ByteArray = bitmap1.bitmapData.getPixels(rect);
var array2:ByteArray = bitmap2.bitmapData.getPixels(rect);
//printConsole("array length :"+array1.length+"array position"+array1.position);
array1.position=0;
array2.position=0;
while(array1.bytesAvailable){
//printConsole("byte checked");
if(array1.readUnsignedInt()==array2.readUnsignedInt()) score++;
}
return (score/(bitmap1.height*bitmap2.width))*100;
}
public function takeSnapshot():Bitmap{
var snapshotHolder:UIComponent = new UIComponent();
var snapshot:BitmapData = new BitmapData(pnlWebcam.video.width, pnlWebcam.video.height, true);
snapshotHolder.y =10;
//pnlSnapshot.addChild(snapshotHolder);
snapshot.draw(pnlWebcam.video);
//applyFilterToBitmap(snapshotbitmap);
pnlSnapshot.visible = true;
var snapshotbitmap:Bitmap = new Bitmap(snapshot);
snapshotHolder.addChild(applyFilterToBitmap(new Bitmap(snapshot)));
pnlSnapshot.addChild(snapshotHolder);
return snapshotbitmap;
false
}
function applyFilterToBitmapData(bitmapdata:BitmapData):BitmapData{
// Create the convolution matrix.
//var matrix:Array = [0, 1, 0,
// 1, -0.5, 1,
// 0, 1, 0];
var matrix:Array = [0, -1, 0,
-1,4, -1,
0, -1, 0];
var convolution:ConvolutionFilter = new ConvolutionFilter();
convolution.matrixX = 3;
convolution.matrixY = 3;
convolution.matrix = matrix;
convolution.divisor = 1;
bitmapdata.applyFilter(bitmapdata,new Rectangle(0,0,pnlWebcam.video.width,pnlWebcam.video.height),new Point(0,0), convolution);
return bitmapdata;
}
function applyFilterToBitmap(bitmap:Bitmap):Bitmap{
// Create the convolution matrix.
var matrix:Array = [0, -1, 0,
-1, 4, -1,
0, -1, 0];
var convolution:ConvolutionFilter = new ConvolutionFilter();
convolution.matrixX = 3;
convolution.matrixY = 3;
convolution.matrix = matrix;
convolution.divisor = 1;
bitmap.filters = [convolution];
return bitmap;
}
function applyFilter():void
{
// Create the convolution matrix.
var matrix:Array = [0, -1, 0,
-1, 4, -1,
0, -1, 0];
var convolution:ConvolutionFilter = new ConvolutionFilter();
convolution.matrixX = 3;
convolution.matrixY = 3;
convolution.matrix = matrix;
convolution.divisor = 1;
pnlWebcam.video.filters = [convolution];
}
public function printConsole(string:String):void{
var oldString:String = consolelabel.text;
consolelabel.text=oldString+":"+string+"\n";
}
private function handlePlayback(event:ItemClickEvent):void{
if(event.currentTarget.selectedValue=="wave"){
Alert.show("Warning, the SineWave Generator has timing issues, isn't easy on the ears and has a tendency to fail stop commands. But at least it's pitch-perfect!");
playBack = "wave";
}
if(event.currentTarget.selectedValue=="piano"){
playBack = "piano";
}
}
private function showHelp(){
dragPanel.visible="true";
var helpText:Text = new Text();
dragPanel.horizontalScrollPolicy="off";
//helpText.maxWidth = 300;
helpText.width=300;
helpText.text = "Welcome to the Camera Music Interpreter.\n\n To interpret sheet music, hold it directly in front of the camera. Make sure to hold the sheet at an appropriate distance (You should be able to see a sharp representation in the Camera Stream pane).\n While the sheet is in view of the camera, hit the Draw Interpreted Lines button. This will make a snapshot and apply the necessary filters. The snapshot will be displayed in the Interpretation pane.\n If the snapshot has too much noise (usually most clip elements are green in that case), lower the noise threshold by moving the slider on top of the screen to the left and hit the Draw Interpreted Lines button again. If the musical staff is mostly green and there is little green outside of the staffs, the noise threshold is correct. Now hit the Interpret Clips button. If the program has deteced a musical staff in the Interpretation image, a button will appear next to that staff, on the right of the Interpretation pane. Click this button to interpret that staff. A represesntation of the staff will be displayed in the Notes pane in the lower right corner. Now, you can play back the interpreted notes by selecting a playback method (the default is piano) and hitting Play. ";
dragPanel.addChild(helpText);
}
//write code here ]]>
</mx:Script>
<mx:RadioButtonGroup id="radiogroup1" itemClick="handlePlayback(event);"/>
<mx:RadioButton x="260" y="0" label="SineWave" groupName="radiogroup1" value="wave" selected="false" enabled="true"/>
<mx:RadioButton x="200" y="0" label="Piano" groupName="radiogroup1" value="piano" selected="true" enabled="true"/>
</mx:Application>
(C) Æliens
27/08/2009
You may not copy or print any of this material without explicit permission of the author or the publisher.
In case of other copyright issues, contact the author.
