diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index 0d5529c..afa508d 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -143,6 +143,11 @@
 	/* Replay detection state at the time we sent the last notification */
 	struct xfrm_replay_state preplay;
 
+	/* internal flag that only holds state for delayed aevent at the
+	 * moment
+	*/
+	u32			xflags;
+
 	/* Replay detection notification settings */
 	u32			replay_maxage;
 	u32			replay_maxdiff;
@@ -168,6 +173,9 @@
 	void			*data;
 };
 
+/* xflags - make enum if more show up */
+#define XFRM_TIME_DEFER	1
+
 enum {
 	XFRM_STATE_VOID,
 	XFRM_STATE_ACQ,
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index a8e14dc..3dc3e1f 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -805,16 +805,22 @@
 	case XFRM_REPLAY_UPDATE:
 		if (x->replay_maxdiff &&
 		    (x->replay.seq - x->preplay.seq < x->replay_maxdiff) &&
-		    (x->replay.oseq - x->preplay.oseq < x->replay_maxdiff))
-			return;
+		    (x->replay.oseq - x->preplay.oseq < x->replay_maxdiff)) {
+			if (x->xflags & XFRM_TIME_DEFER)
+				event = XFRM_REPLAY_TIMEOUT;
+			else
+				return;
+		}
 
 		break;
 
 	case XFRM_REPLAY_TIMEOUT:
 		if ((x->replay.seq == x->preplay.seq) &&
 		    (x->replay.bitmap == x->preplay.bitmap) &&
-		    (x->replay.oseq == x->preplay.oseq))
+		    (x->replay.oseq == x->preplay.oseq)) {
+			x->xflags |= XFRM_TIME_DEFER;
 			return;
+		}
 
 		break;
 	}
@@ -825,8 +831,10 @@
 	km_state_notify(x, &c);
 
 	if (x->replay_maxage &&
-	    !mod_timer(&x->rtimer, jiffies + x->replay_maxage))
+	    !mod_timer(&x->rtimer, jiffies + x->replay_maxage)) {
 		xfrm_state_hold(x);
+		x->xflags &= ~XFRM_TIME_DEFER;
+	}
 }
 EXPORT_SYMBOL(xfrm_replay_notify);
 
@@ -836,10 +844,15 @@
 
 	spin_lock(&x->lock);
 
-	if (xfrm_aevent_is_on() && x->km.state == XFRM_STATE_VALID)
-		xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT);
+	if (x->km.state == XFRM_STATE_VALID) {
+		if (xfrm_aevent_is_on())
+			xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT);
+		else
+			x->xflags |= XFRM_TIME_DEFER;
+	}
 
 	spin_unlock(&x->lock);
+	xfrm_state_put(x);
 }
 
 int xfrm_replay_check(struct xfrm_state *x, u32 seq)