Code to extract music notation from Flowkey app

% Matlab code to extract music notation from flowkey screen grab video . See https://youtu.be/4oOMWkMRCCg

v = VideoReader('takemetochurch.mp4');
ind = 5*25;
frame =v.read(ind);
im = frame(430:660 ,: ,:);
new_im = rgb2gray(im);
prev_x = sum(new_im); % used to find bes overlap of frames

while 1
    ind = ind+5*25;
    if(ind > v.NumFrames)
        break
    end
    frame =v.read(ind);
    im = frame(430:660 ,: ,:);
    img = rgb2gray(im); %conver to greyscale
    x = sum(img);
    xc =[]; %stores cross correlation result
    seg2 = prev_x(end-600+1:end); %end of previous frame
    for w = 1: 300
        seg1 = x(1+w:600+w);
        xc(w) = sum(seg1.*seg2)./(sqrt(sum(seg1.^2)*sum(seg2.^2))); %cross correlation
    end
    [val loc] = max(xc); % loc is where the max correlation occurs
    
    [ r c d] = size(im);
    new_seg =img(:,201:end);
    [ r1 c1 d1] = size(new_seg);
    new_im(:,end-600+1-loc+200:end-600-loc+200+c1) = new_seg;
    prev_x = x;
    
end
img = new_im;
% find where the bars occur in the image
for k =1: length(img)
    x(k) = sum(img(95:145,k)); %the bar lines cross are clear from rows 75 to 125
end
bars = find(x<5000); %obtained value of 5000 by visually inspecting plot(x)


op_image_max_width = 2000;
op_image_max_height = 1200;

overlap = 10;
start_pt = overlap+1;
prev_end_pt = bars(1);
op_img = img(1,1);
k = 1;
img_num = 1;
while(k<length(bars))
    end_pt = bars(k);
    if(end_pt-start_pt>op_image_max_width)
        [rows cols ] = size(img(:,start_pt-overlap:prev_end_pt+overlap));
        [rows1 cols1] = size(op_img);
        if(rows+rows1 > op_image_max_height)
            imwrite(op_img, ['notation_' num2str(img_num) '.jpg']);
            op_img = img(1,1);
            
            img_num = img_num + 1;
        end
        op_img(end+1:end+rows,1:cols,:) = img(:,start_pt-10:prev_end_pt+10);
        start_pt = prev_end_pt;
    end
    prev_end_pt = end_pt;
    k = k + 1;
end

imwrite(op_img, ['notation_' num2str(img_num) '.jpg']);

Advertisement