I’ve integrated iOS MotionManager into the MidiKeyboardComponent to pick up touch velocity on the iPad. It’s not bad, but there’s a fair amount of unwanted latency, especially when compared to Garageband’s keyboard velocity, which is one of the better feeling velocity keyboards I’ve come across. I thought the info and code below might be useful for anyone else looking to do this, and I’m hoping for suggestions on how one might improve this.
I had originally hoped to add touch sensitive velocity to the MidiKeyboardComponent on iPad using 3D Touch. Turns out, Apple hasn’t added 3D touch to the iPad. After snooping around, it’s widely believed this is achieved in apps like Garageband, via the Accelerometer.
Upon initial implementation, I realized that the dip in the z plane happens after the initial touch. So, in mouseDown, I collected a series of subsequent Accelerometer-z values, took the lowest one, and then triggered the note. This worked, however, in doing so, I created a quite a bit of latency, making the keyboard hard to play. This is due to the time it takes to collect the values, and possibly the time to copy the MouseEvent (a hacky way to hold onto the event).
So I switched to the Gyroscope’s y axis, which works only because the keybaord is on the bottom of the screen, and I found I was able to pick up the necessary values quicker, shortening the latency. Though it’s still more latent than I’d like, and certainly more latent than Garageband. And, with the Gyroscope, the iPad has to be on a surface with a little give (like a pillow) in order to allow for rotation. So…
Accelerometer = flat surface, too much latency
Gyroscope = soft surface, latency I can live with but would rather not
MotionManager.h
//
// MotionManager.h
// Syntorial
//
// Created by Joe Hanley on 1/28/15.
//
//
#define MaxYCount 3
class MyKeyboardComponent;
class MotionManager
{
public:
MotionManager(MyKeyboardComponent* keyboard_);
~MotionManager();
void gyroChanged (float y);
void accelerometerChanged (float z);
void startCollecting();
void start();
void stop();
private:
void* motionManagerWrapper;
MyKeyboardComponent* keyboard;
bool isCollecting, isRunning;
float highestY;
int yCount;
};
MotionManager.mm
#include "MotionManager.h"
#import <UIKit/UIKit.h>
#import <CoreMotion/CoreMotion.h>
#import <CoreLocation/CoreLocation.h>
@interface MotionManagerWrapper : NSObject
{
MotionManager* owner;
}
@property (strong, nonatomic) CMMotionManager *motionManager;
@end
@implementation MotionManagerWrapper
- (void) startMotionManager
{
self.motionManager = [[CMMotionManager alloc] init];
self.motionManager.accelerometerUpdateInterval = .001;
self.motionManager.gyroUpdateInterval = .001;
// [self.motionManager startAccelerometerUpdatesToQueue:[NSOperationQueue currentQueue]
// withHandler:^(CMAccelerometerData *accelerometerData, NSError *error) {
// [self outputAccelerationData:accelerometerData.acceleration];
// if(error){
//
// NSLog(@"%@", error);
// }
// }];
[self.motionManager startGyroUpdatesToQueue:[NSOperationQueue currentQueue]
withHandler:^(CMGyroData *gyroData, NSError *error) {
[self outputRotationData:gyroData.rotationRate];
}];
}
- (void) stopMotionManager
{
[self.motionManager stopGyroUpdates];
// [self.motionManager stopAccelerometerUpdates];
self.motionManager = nil;
}
-(void)outputAccelerationData:(CMAcceleration)acceleration
{
owner->accelerometerChanged(acceleration.z);
}
-(void)outputRotationData:(CMRotationRate)rotation
{
owner->gyroChanged(fabs(rotation.y));
}
- (id) initWithOwner: (MotionManager*) owner_
{
if ((self = [super init]) != nil)
{
owner = owner_;
};
return self;
}
- (void) dealloc
{
[self stopMotionManager];
[super dealloc];
}
@end
//==============================================================================
MotionManager::MotionManager(MyKeyboardComponent* keyboard_)
: keyboard(keyboard_),
isCollecting(false),
isRunning(false),
highestY (0.0),
yCount (0)
{
MotionManagerWrapper* newManager = [[MotionManagerWrapper alloc] initWithOwner: this];
[newManager retain];
motionManagerWrapper = newManager;
}
MotionManager::~MotionManager()
{
[((MotionManagerWrapper*) motionManagerWrapper) release];
}
void MotionManager::accelerometerChanged (float z)
{
// if (isCollecting)
// {
// if (z < lowestZ)
// lowestZ = z;
// zCount++;
//
// if (zCount >= MaxYCount)
// {
// isCollecting = false;
// float velocity = 1;
// float high = -0.9, low = -1.3;
// lowestZ = jlimit(low, high, lowestZ);
// float range = high - low;
// velocity = (lowestZ - high) / range * -1;
// keyboard->triggerNoteWithVelocity(velocity);
// }
// }
}
void MotionManager::gyroChanged (float y)
{
if (isCollecting)
{
if (y > highestY)
highestY = y;
yCount++;
if (yCount >= MaxYCount)
{
isCollecting = false;
if (highestY < 0.1)
highestY = 0.1;
keyboard->triggerNoteWithVelocity(highestY);
}
}
}
void MotionManager::startCollecting()
{
yCount = 0;
highestY = 0.0;
// zCount = 0;
// lowestZ = 0;
isCollecting = true;
}
void MotionManager::start()
{
if (!isRunning)
{
[(MotionManagerWrapper*) motionManagerWrapper startMotionManager];
isRunning = true;
}
}
void MotionManager::stop()
{
[(MotionManagerWrapper*) motionManagerWrapper stopMotionManager];
isRunning = false;
}
MyKeyboardComponent.h (abbreviated for relevancy)
class MyKeyboardComponent : public MidiKeyboardComponent
{
public:
MyKeyboardComponent(MidiKeyboardState& state_, const Orientation orientation_);
~MyKeyboardComponent();
void mouseDown(const MouseEvent &event);
void fillMouseStats(const MouseEvent& event, bool wasDragged);
void triggerNote(const MouseEvent& event, float velocity = -1.0);
void triggerNoteWithVelocity(float velocity);
private:
Orientation orientation;
ScopedPointer<MotionManager> motionManager;
class MouseStats
{
public:
MouseStats()
: source (nullptr)
{}
ScopedPointer<MouseInputSource> source;
Point<float> position;
ModifierKeys modifiers;
float pressure;
float orientation;
float rotation;
float tiltX;
float tiltY;
Component* eventComponent;
Component* originator;
Time eventTime;
Point<float> mouseDownPos;
Time mouseDownTime;
int numberOfClicks;
bool mouseWasDragged;
};
MouseStats lastMouseDownStats;
};
MyKeyboardComponent.cpp (abbreviated for relevancy)
MyKeyboardComponent::MyKeyboardComponent(MidiKeyboardState& state_, const Orientation orientation_)
: MidiKeyboardComponent(state_, orientation_),
orientation (orientation_),
doVelocity (false)
{
lastMouseDownStats.source = new MouseInputSource(Desktop::getInstance().getMainMouseSource());
lastMouseDownStats.position.setX(-999);
motionManager = new MotionManager(this);
}
MyKeyboardComponent::~MyKeyboardComponent()
{
motionManager->stop();
}
void MyKeyboardComponent::mouseDown(const MouseEvent &event)
{
fillMouseStats(event, false);
motionManager->startCollecting();
}
void MyKeyboardComponent::fillMouseStats(const MouseEvent& event, bool wasDragged)
{
*(lastMouseDownStats.source) = event.source;
lastMouseDownStats.position.setX(event.position.getX());
lastMouseDownStats.position.setY(event.position.getY());
lastMouseDownStats.modifiers = event.mods;
lastMouseDownStats.pressure = event.pressure;
lastMouseDownStats.orientation = event.orientation;
lastMouseDownStats.rotation = event.rotation;
lastMouseDownStats.tiltX = event.tiltX;
lastMouseDownStats.tiltY = event.tiltY;
lastMouseDownStats.eventComponent = event.eventComponent;
lastMouseDownStats.originator = event.originalComponent;
lastMouseDownStats.eventTime = event.eventTime;
lastMouseDownStats.mouseDownPos.setX(event.getMouseDownX());
lastMouseDownStats.mouseDownPos.setY(event.getMouseDownY());
lastMouseDownStats.mouseDownTime = event.mouseDownTime;
lastMouseDownStats.numberOfClicks = event.getNumberOfClicks();
lastMouseDownStats.mouseWasDragged = wasDragged;
}
void MyKeyboardComponent::triggerNoteWithVelocity(float velocity)
{
MouseEvent eventCopy (*(lastMouseDownStats.source), lastMouseDownStats.position, lastMouseDownStats.modifiers, lastMouseDownStats.pressure, lastMouseDownStats.orientation, lastMouseDownStats.rotation, lastMouseDownStats.tiltX, lastMouseDownStats.tiltY, lastMouseDownStats.eventComponent, lastMouseDownStats.originator, lastMouseDownStats.eventTime, lastMouseDownStats.mouseDownPos, lastMouseDownStats.mouseDownTime, lastMouseDownStats.numberOfClicks, lastMouseDownStats.mouseWasDragged);
triggerNote(eventCopy, velocity);
}