Ok I fixed it by adding this before the code I pasted above
$custom_mail = get_post_meta($post->ID, 'custom_email', true);
if ($custom_mail != '') {
wp_mail( $custom_mail, __('Contact "', 'appthemes').$post->post_title.'"', $message, $headers, $attachments );
}
