then add this code in your wplms-customizer.php file in wplms customizer plugin :
add_action('wp_footer','check_logged_in_and_form');
function check_logged_in_and_form(){
global $post;
$course_id = $post->ID;
if(is_singular('course')){
if((!is_user_logged_in()) || (is_user_logged_in() && function_exists('bp_course_is_member') && !bp_course_is_member($course_id,get_current_user_id()))){
?>
<script>
jQuery(document).ready(function($){
$('a.course_button').on('click',function(event){
event.preventDefault();
$('.ajax-popup-link').trigger('click');
});
});
</script>
<?php
}
}
}