Navigation Problem

I’m having problems with navigation trough posts on a page with posts from a certain category. I displays the latests 3 posts, but when I click the ‘Vorige’ or ‘Previous Posts’ button I does go to /page-2 but it keeps displaying the latest 3 posts.

Here is my code for the Blog based page, could anybody be kind enough to help out?

<?php
/*
Template Name: Blog
*/
?>
<?php get_header(); ?>

<?php query_posts('cat=-4'); ?>

<?php if ( have_posts() ) : ?>
	<h2><?php _e( 'Not Found', 'framework' ); ?></h2>

	<p><?php _e( 'Apologies, but no results were found for the requested archive. Perhaps searching will help find a related post.', 'framework' ); ?></p>
<?php endif; ?>

<?php while ( have_posts() ) : the_post(); ?>

		<div class="hentry-outer">
			<div <?php post_class(); ?>>
				<div class="hentry-header">
					<h1><a href="<?php the_permalink(); ?>"><?php the_title(); ?></a></h1>

					<!-- <a href="<?php the_permalink(); ?>#comments" class="meta-comments"><?php comments_number('0','1','%'); ?></a> -->

					<div class="meta"><span><?php echo get_the_date(); ?> // <?php the_category(', '); ?></span></div>
				</div><!-- /hentry-header -->

				<?php
					if (( function_exists('has_post_thumbnail')) && ( of_get_option('featured_image') == '1' )) : ?>
					<div class="featured-image">
						<a href="<?php the_permalink(); ?>"><?php the_post_thumbnail('post-thumb'); ?></a>
					</div>
				<?php endif; ?>

                <?php global $more; $more = 0; ?>

				<?php the_content( __( 'Lees verder', 'framework' ) .' &rarr;' ); ?>

				<?php wp_link_pages( array( 'before' => '' . __( 'Pages:', 'framework' ), 'after' => '' ) ); ?>

				<?php
					if (( of_get_option('show_tags') == '1' )) :
					the_tags('<ul class="meta-tags"><li>','</li><li>','</li></ul>');
					endif;
				?>

				<?php comments_template( '', true ); ?>
			</div><!-- /hentry -->
		</div><!-- /hentry-outer -->

<?php endwhile; ?>

<?php if (  $wp_query->max_num_pages > 1 ) : ?>
	<ul id="page-navi">
		<li><?php next_posts_link( __( 'Vorige', 'framework' ) ); ?></li>
		<li><?php previous_posts_link( __( 'Volgende', 'framework' ) ); ?></li>
	</ul>
<?php endif; ?>
<?php get_sidebar(); ?>
<?php get_footer(); ?>